feat(go-services): tracker WS servers (/ws/position + /ws/live) + robust shadow
Completes the Go tracker as a cutover-ready drop-in: - wslive.go: browser broadcast hub with per-client subscribe filters (nil=all), request_dungeon_map replies, and command routing; auth = internal-trust or session cookie. The ingestor broadcasts every handled event to it. - wsposition.go: plugin ingest server with X-Plugin-Secret/SHARED_SECRET auth (constant-time, fails closed, legacy fallback), register -> plugin_conns, and dispatch into the shared Ingestor. plugin registry for backend->plugin commands. - main.go: statusRecorder.Unwrap() so coder/websocket can hijack through the logging middleware (WS handshakes failed without it); /ws/ bypasses HTTP auth. Shadow consumer robustness (the harness was being evicted under the full firehose): decouple socket read from processing — the read loop only copies raw frames to a queue; a worker unmarshals + dispatches. JSON parsing in the read loop was slowing it enough that Python's broadcast send errored and evicted us (Read then blocked forever). Added a 25s read-deadline watchdog to self-heal. Validated live: shadow /live online = 73 = production; telemetry sustained ~12/s, 0 drops, no eviction; and the shadow's /ws/live re-broadcast stream is IDENTICAL to production's (TOTAL 2150=2150, every event type exact). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
7350b00341
commit
27757636e4
7 changed files with 418 additions and 34 deletions
|
|
@ -46,39 +46,75 @@ func (s *Server) shadowConnect(ctx context.Context, wsURL string) error {
|
|||
defer c.CloseNow()
|
||||
c.SetReadLimit(32 << 20) // nearby_objects / dungeon_map payloads can be large
|
||||
|
||||
// Keepalive pings.
|
||||
pingCtx, cancel := context.WithCancel(ctx)
|
||||
connCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
// Keepalive pings.
|
||||
go func() {
|
||||
t := time.NewTicker(20 * time.Second)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-pingCtx.Done():
|
||||
case <-connCtx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
pc, cc := context.WithTimeout(pingCtx, 10*time.Second)
|
||||
pc, cc := context.WithTimeout(connCtx, 10*time.Second)
|
||||
_ = c.Ping(pc)
|
||||
cc()
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Decouple socket read from ALL processing, including JSON parsing: the read
|
||||
// loop only copies raw frames onto a queue, so it drains the socket as fast
|
||||
// as the network delivers. If parsing or DB-bound dispatch ran inline, the
|
||||
// read would stall, the upstream /ws/live broadcast send would error, and
|
||||
// Python would evict us (Read then blocks forever). A single worker
|
||||
// unmarshals + dispatches in order, preserving per-char kill-delta / combat
|
||||
// accumulation.
|
||||
queue := make(chan []byte, 16384)
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
for raw := range queue {
|
||||
var m map[string]any
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
continue
|
||||
}
|
||||
s.ingestor.dispatch(connCtx, m)
|
||||
}
|
||||
}()
|
||||
|
||||
s.log.Info("shadow consumer connected; replaying /ws/live into ingest", "url", wsURL)
|
||||
var n int
|
||||
var n, dropped int
|
||||
loopErr := s.shadowReadLoop(ctx, c, queue, &n, &dropped)
|
||||
close(queue)
|
||||
<-done
|
||||
return loopErr
|
||||
}
|
||||
|
||||
func (s *Server) shadowReadLoop(ctx context.Context, c *websocket.Conn, queue chan []byte, n, dropped *int) error {
|
||||
for {
|
||||
_, raw, err := c.Read(ctx)
|
||||
// Read deadline acts as a liveness watchdog: the firehose is constant, so
|
||||
// a long silence means the upstream evicted us without closing — time out
|
||||
// and let runShadowConsumer reconnect.
|
||||
rctx, rcancel := context.WithTimeout(ctx, 25*time.Second)
|
||||
_, raw, err := c.Read(rctx)
|
||||
rcancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var m map[string]any
|
||||
if json.Unmarshal(raw, &m) != nil {
|
||||
continue
|
||||
select {
|
||||
case queue <- raw:
|
||||
default:
|
||||
*dropped++
|
||||
if *dropped%1000 == 1 {
|
||||
s.log.Warn("shadow queue full; dropping messages", "dropped", *dropped)
|
||||
}
|
||||
}
|
||||
s.ingestor.dispatch(ctx, m)
|
||||
n++
|
||||
if n%5000 == 0 {
|
||||
s.log.Info("shadow consumer progress", "messages", n)
|
||||
*n++
|
||||
if *n%5000 == 0 {
|
||||
s.log.Info("shadow consumer progress", "messages", *n, "queued", len(queue), "dropped", *dropped)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue