feat(go-services): Phase 2 ingest — shared Ingestor + shadow consumer
Implements the plugin event handlers (the /ws/position write logic) as a shared Ingestor, validated against real traffic by replaying Python's /ws/live firehose into an isolated dereth_go DB (no production write, no plugin stolen). - ingest.go: faithful ports of telemetry (kill-delta -> char_stats, server received_at stamp), rare (rare_stats/rare_stats_sessions/rare_events), portal (coord upsert), character_stats (stats_data JSONB subset + upsert), spawn, and the memory-only handlers (vitals/quest/equipment_cantrip/nearby/dungeon). In -memory live state + read-side overlay accessors. - shadow.go: coder/websocket consumer of /ws/live -> Ingestor.dispatch (telemetry matched by shape since its broadcast has no type field). - main.go/store.go: ingest mode (READ_ONLY=false + SHADOW_INGEST_WS) wires the ingestor; read handlers (/character-stats, /equipment-cantrip, /quest-status) now consult the live overlay first, like Python. - compose: shadow instance ingests ws://dereth-tracker:8765/ws/live. Validated live: dereth_go has 73 distinct telemetry chars; shadow /live online set == production (73=73); character_stats 5/5 exact byte-match (0 mismatch); char_stats kill-deltas + portals accumulating. compare/compare_ingest.py. Deferred to next pass: combat_stats (delta/merge), share_*, the /ws/position + /ws/live servers (for cutover). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
6a839e69bc
commit
a5d69ba88d
7 changed files with 621 additions and 3 deletions
64
go-services/compare/compare_ingest.py
Normal file
64
go-services/compare/compare_ingest.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Validate the Go shadow ingest (dereth_go) against production (dereth).
|
||||
|
||||
Run on the server. The shadow tracker replays Python's /ws/live firehose into
|
||||
its own dereth_go DB. Absolute counts differ (shadow started fresh; char_stats /
|
||||
rare_stats accumulate deltas from connect time), so we validate the paths whose
|
||||
writes are FULL upserts/inserts and therefore exactly comparable:
|
||||
|
||||
* character_stats: a full-payload upsert. For a character whose row has the
|
||||
SAME timestamp in both DBs, stats_data must be byte-identical.
|
||||
* /live online set: telemetry end-to-end (compared separately by the caller).
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
SEP = "\x1f"
|
||||
|
||||
|
||||
def q(container, db, sql):
|
||||
out = subprocess.check_output(
|
||||
["docker", "exec", container, "psql", "-U", "postgres", "-d", db, "-tA", "-F", SEP, "-c", sql],
|
||||
text=True)
|
||||
return [line.split(SEP) for line in out.splitlines() if line.strip()]
|
||||
|
||||
|
||||
def main():
|
||||
print("=== dereth_go ingested row counts ===")
|
||||
counts = q("dereth-go-db", "dereth_go", """
|
||||
SELECT 'telemetry_events', count(*)::text FROM telemetry_events
|
||||
UNION ALL SELECT 'telemetry_distinct_chars', count(distinct character_name)::text FROM telemetry_events
|
||||
UNION ALL SELECT 'character_stats', count(*)::text FROM character_stats
|
||||
UNION ALL SELECT 'char_stats', count(*)::text FROM char_stats
|
||||
UNION ALL SELECT 'rare_events', count(*)::text FROM rare_events
|
||||
UNION ALL SELECT 'rare_stats', count(*)::text FROM rare_stats
|
||||
UNION ALL SELECT 'portals', count(*)::text FROM portals
|
||||
""")
|
||||
for k, v in counts:
|
||||
print(f" {k:26} {v}")
|
||||
|
||||
print("\n=== character_stats exact match (same-timestamp rows) ===")
|
||||
prod = {r[0]: (r[1], r[2]) for r in
|
||||
q("dereth-db", "dereth", "SELECT character_name, timestamp::text, stats_data::text FROM character_stats")}
|
||||
shadow = q("dereth-go-db", "dereth_go",
|
||||
"SELECT character_name, timestamp::text, stats_data::text FROM character_stats")
|
||||
match = mismatch = newer = 0
|
||||
for name, ts, sd in shadow:
|
||||
if name not in prod:
|
||||
continue
|
||||
pts, psd = prod[name]
|
||||
if ts != pts:
|
||||
newer += 1 # one side got a newer character_stats message; not comparable
|
||||
continue
|
||||
if json.loads(sd) == json.loads(psd):
|
||||
match += 1
|
||||
else:
|
||||
mismatch += 1
|
||||
print(f" MISMATCH {name}")
|
||||
print(f" exact match={match} mismatch={mismatch} skipped(diff timestamp)={newer}")
|
||||
print("\nRESULT:", "ingest OK" if mismatch == 0 else f"{mismatch} character_stats mismatch(es)")
|
||||
return 1 if mismatch else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Loading…
Add table
Add a link
Reference in a new issue