new comments

This commit is contained in:
erik 2025-05-24 18:33:03 +00:00
parent b2f649a489
commit 09404da121
13 changed files with 430 additions and 70 deletions

View file

@ -1,3 +1,8 @@
"""Asynchronous database layer for telemetry service using PostgreSQL/TimescaleDB.
Defines table schemas via SQLAlchemy Core and provides an
initialization function to set up TimescaleDB hypertable.
"""
import os
import sqlalchemy
from databases import Database
@ -8,10 +13,13 @@ DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:password@localho
# Async database client
database = Database(DATABASE_URL)
# Metadata for SQLAlchemy Core
# SQLAlchemy metadata container for table definitions
metadata = MetaData()
# Telemetry events hypertable schema
# --- Table Definitions ---
# Table for storing raw telemetry snapshots at scale (converted to hypertable)
telemetry_events = Table(
# Time-series hypertable storing raw telemetry snapshots from plugins
"telemetry_events",
metadata,
Column("id", Integer, primary_key=True),
@ -36,17 +44,18 @@ telemetry_events = Table(
Column("latency_ms", Float, nullable=True),
)
# Persistent kill statistics per character
# Persistent kill statistics per character
# Table for persistent total kills per character
char_stats = Table(
# Stores cumulative kills per character in a single-row upsert table
"char_stats",
metadata,
Column("character_name", String, primary_key=True),
Column("total_kills", Integer, nullable=False, default=0),
)
# Rare event tracking: total and per-session counts
# Table for persistent total rare counts per character
rare_stats = Table(
# Stores cumulative rare event counts per character
"rare_stats",
metadata,
Column("character_name", String, primary_key=True),
@ -54,14 +63,16 @@ rare_stats = Table(
)
rare_stats_sessions = Table(
# Stores per-session rare counts; composite PK (character_name, session_id)
"rare_stats_sessions",
metadata,
Column("character_name", String, primary_key=True),
Column("session_id", String, primary_key=True),
Column("session_rares", Integer, nullable=False, default=0),
)
# Spawn events: record mob spawns for heatmapping
# Table for recording spawn events (mob creates) for heatmap analysis
spawn_events = Table(
# Records individual mob spawn occurrences for heatmap and analysis
"spawn_events",
metadata,
Column("id", Integer, primary_key=True),
@ -72,8 +83,9 @@ spawn_events = Table(
Column("ns", Float, nullable=False),
Column("z", Float, nullable=False),
)
# Rare events: record individual rare spawns for future heatmaps
# Table for recording individual rare spawn events for analysis
rare_events = Table(
# Records individual rare mob events for detailed analysis and heatmaps
"rare_events",
metadata,
Column("id", Integer, primary_key=True),
@ -86,23 +98,32 @@ rare_events = Table(
)
async def init_db_async():
"""Create tables and enable TimescaleDB hypertable for telemetry_events."""
"""Initialize PostgreSQL/TimescaleDB schema and hypertable.
Creates all defined tables and ensures the TimescaleDB extension is
installed. Converts telemetry_events table into a hypertable for efficient
time-series data storage.
"""
# Create tables in Postgres
engine = sqlalchemy.create_engine(DATABASE_URL)
# Reflects metadata definitions into actual database tables via SQLAlchemy
metadata.create_all(engine)
# Enable TimescaleDB extension and convert telemetry_events to hypertable
# Use a transactional context to ensure DDL statements are committed
with engine.begin() as conn:
# Enable or update TimescaleDB extension
# Install or confirm TimescaleDB extension to support hypertables
try:
conn.execute(text("CREATE EXTENSION IF NOT EXISTS timescaledb"))
except Exception as e:
print(f"Warning: failed to create extension timescaledb: {e}")
# Update TimescaleDB extension if an older version exists
try:
conn.execute(text("ALTER EXTENSION timescaledb UPDATE"))
except Exception as e:
print(f"Warning: failed to update timescaledb extension: {e}")
# Create hypertable for telemetry_events, skip default indexes to avoid collisions
# Transform telemetry_events into a hypertable partitioned by timestamp
try:
conn.execute(text(
"SELECT create_hypertable('telemetry_events', 'timestamp', \