Initial commit — leak-hunt project complete

Five bugs identified and patched in retail Asheron's Call client:
- v3b: palette refcount over-increment (3-byte NOP at two sites)
- v5: RenderSurface PurgeResource no-op stub (vtable slot 2 thunk)
- v11: two dangling-pointer crash guards (NULL-check + reorder)
- v14: CEnvCell::Destroy ClipPlaneList leak (18-byte JMP to cleanup thunk)
- v22: unpacker stale-pointer SEH guard (whole-function __try/__except)

All five ship in leakfix.dll (117 KB, SHA d282f23c…) which is loaded
by acclient.exe at process start via PE import table patching by
tools/install_leakfix.py.

Controlled 15-client fleet soak: unpatched control died at 26h with
palette exhaustion; all 14 patched clients survived past that point
and reached ≥5-day uptime.

Residual ~15 MB/h growth traced to d3d9.dll's internal slab allocator
(260KB surface backing buffers retained after Release). See REPORT.md
§10 for the full investigation; conclusion is that it's unfixable from
outside d3d9.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
acbot 2026-05-23 21:05:17 +02:00
commit 57b5e43d0e
199 changed files with 1648333 additions and 0 deletions

533
tools/dashboard.py Normal file
View file

@ -0,0 +1,533 @@
"""dashboard.py — local fleet leak dashboard.
Serves on http://localhost:8080.
/ -> HTML page with charts (WS + per-class instance counts)
/data.json -> parsed snapshot data from artifacts/snapshots/main.tsv
"""
import http.server
import json
import socketserver
import threading
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
TSV = ROOT / "artifacts" / "snapshots" / "main.tsv"
REPORT_MD = ROOT / "REPORT.md"
PORT = 8080
# Expected column order written by snapshot_compare.py
HEADER_COLS = ["timestamp", "client", "label", "pid", "mb",
"uiitem", "uiitem_cleared", "uiitem_active",
"palette", "cphysicsobj",
"renderSurf", "renderSurfD3D", "renderTexD3D",
"csurface", "imgtex", "cgfxobj", "d3dxmesh"]
# Metrics we want to chart, in display order
METRICS = ["mb", "palette", "renderSurf", "renderSurfD3D",
"renderTexD3D", "csurface", "imgtex", "cphysicsobj",
"uiitem"]
METRIC_LABELS = {
"mb": "Memory (MB)",
"palette": "Palette instances",
"renderSurf": "RenderSurface base instances",
"renderSurfD3D": "RenderSurfaceD3D instances",
"renderTexD3D": "RenderTextureD3D instances",
"csurface": "CSurface instances",
"imgtex": "ImgTex instances",
"cphysicsobj": "CPhysicsObj instances",
"uiitem": "UIElement_UIItem instances",
}
def load_tsv():
"""Parse main.tsv into {client: [(ts, {metric:val,...}), ...]}.
Rows where mb == 0 or 'DEAD' are skipped (stale-PID rows).
Header row (with literal 'timestamp' in col 1) is skipped.
"""
out = {}
if not TSV.exists():
return out
with open(TSV) as f:
for line in f:
line = line.rstrip("\n")
if not line: continue
parts = line.split("\t")
if len(parts) < 5: continue
if parts[0] == "timestamp": continue # header
row = {}
for i, col in enumerate(HEADER_COLS):
if i >= len(parts): break
row[col] = parts[i]
# Skip dead-PID rows
mb_str = row.get("mb", "")
if mb_str in ("0", "DEAD", "NOACCESS", ""): continue
try:
ts = row["timestamp"]
client = row["client"]
metrics = {}
for m in METRICS:
v = row.get(m, "")
if v == "": continue
try: metrics[m] = int(v)
except ValueError: pass
# also include pid so the dashboard can detect restarts
pid_str = row.get("pid", "")
if pid_str:
try: metrics["pid"] = int(pid_str)
except ValueError: pass
out.setdefault(client, []).append((ts, metrics))
except KeyError:
continue
# Sort each client's series by timestamp
for c in out:
out[c].sort(key=lambda x: x[0])
# Filter to rows matching the CURRENT (latest) PID per character so we
# don't conflate pre-restart series with post-restart series under the
# same character name. (Old PIDs become DEAD rows but were once alive
# and have data we'd otherwise plot as a misleading plateau.)
filtered = {}
for c, series in out.items():
if not series: continue
latest_pid = series[-1][1].get("pid")
if latest_pid is None:
filtered[c] = series
continue
kept = [(ts, m) for ts, m in series if m.get("pid") == latest_pid]
if kept: filtered[c] = kept
return filtered
HTML = """<!doctype html>
<html><head>
<meta charset="utf-8">
<title>Leak-Hunt Fleet Dashboard</title>
<script src="https://cdn.jsdelivr.net/npm/chart.js@4.4.0/dist/chart.umd.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/chartjs-adapter-date-fns@3.0.0/dist/chartjs-adapter-date-fns.bundle.min.js"></script>
<style>
body { font-family: system-ui, sans-serif; margin: 0; padding: 16px; background:#0f1419; color:#e6e6e6; }
h1 { margin: 0 0 8px 0; font-size: 18px; }
.meta { color: #888; font-size: 12px; margin-bottom: 16px; }
.grid { display: grid; grid-template-columns: 1fr 1fr; gap: 16px; }
.chart-card { background: #1a1f29; border: 1px solid #2a3140; padding: 12px; border-radius: 6px; }
.chart-card h2 { margin: 0 0 8px 0; font-size: 13px; color: #ccc; font-weight: 500; }
canvas { height: 260px !important; }
#status { font-size: 11px; color: #888; }
button { background:#243044; color:#eee; border:1px solid #3a4458; padding:4px 10px; cursor:pointer; border-radius:3px; font-size:11px; }
button:hover { background:#2e3c54; }
.controls { margin-bottom: 12px; }
table.summary { border-collapse: collapse; width: 100%; font-size: 11px; margin-bottom: 16px; }
table.summary th, table.summary td { border: 1px solid #2a3140; padding: 4px 8px; text-align: right; }
table.summary th { background: #1a1f29; cursor: pointer; user-select: none; color: #ccc; font-weight: 500; }
table.summary th:hover { background: #243044; }
table.summary td.client { text-align: left; font-weight: 500; }
table.summary tr:nth-child(even) td { background: #15191f; }
table.summary tr.control td { background: #2a1818 !important; color: #ffc; }
table.summary tr.danger td.cap { color: #ff5050; font-weight: bold; }
table.summary tr.warn td.cap { color: #ffaa00; }
table.summary tr.ok td.cap { color: #50e0a0; }
.grow-rate-pos { color: #ff8888; }
.grow-rate-zero { color: #888; }
.pill { display: inline-block; padding: 2px 6px; border-radius: 3px; font-size: 10px; margin-left: 6px; }
.pill-ctrl { background: #4a1818; color: #ffc; }
</style>
</head><body>
<h1>Leak-Hunt Fleet Dashboard <a href="/report" style="font-size:13px;font-weight:normal;color:#5da5da;text-decoration:none;margin-left:16px;">[Final Report ]</a></h1>
<div class="meta">
<span id="status">loading...</span>
&nbsp;|&nbsp;
<span>data: <code>artifacts/snapshots/main.tsv</code></span>
&nbsp;|&nbsp;
<button onclick="loadData()">refresh</button>
&nbsp;|&nbsp;
<button onclick="toggleAutoRefresh()" id="autoBtn">auto-refresh: off</button>
&nbsp;|&nbsp;
<button onclick="setRange(0)">all data</button>
<button onclick="setRange(24)">last 24h</button>
<button onclick="setRange(6)">last 6h</button>
</div>
<h2 style="font-size:13px;margin:16px 0 6px 0;color:#aaa;">Current state &nbsp;<span style="color:#666;font-weight:normal;font-size:11px;">(click headers to sort; growth rate = avg per hour from last 4 snapshots; cap = projected hours until 2048MB at current rate)</span></h2>
<div id="summaryWrap"><table class="summary" id="summary"></table></div>
<h2 style="font-size:13px;margin:16px 0 6px 0;color:#aaa;">Charts &nbsp;<span style="color:#666;font-weight:normal;font-size:11px;">(click legend names to toggle clients)</span></h2>
<div class="grid" id="charts"></div>
<script>
const METRICS = __METRICS__;
const METRIC_LABELS = __METRIC_LABELS__;
const PALETTE = [
"#5da5da","#f17cb0","#b2912f","#b276b2","#decf3f","#f15854",
"#60bd68","#faa43a","#4d4d4d","#9c27b0","#00bcd4","#ff9800",
"#cddc39","#e91e63","#3f51b5"
];
const CAP_MB = 2048;
let charts = {};
let autoTimer = null;
let rangeHours = 0; // 0 = all
let sortKey = "mb";
let sortDir = -1; // -1 desc, +1 asc
function pickColor(client, idx) {
if (client.toLowerCase().includes("jerry")) return "#ff3030"; // control = red
return PALETTE[idx % PALETTE.length];
}
function parseTs(ts) { return new Date(ts.replace(" ", "T")).getTime(); }
function filterByRange(series) {
if (rangeHours <= 0 || series.length === 0) return series;
const cutoff = parseTs(series[series.length-1][0]) - rangeHours * 3600 * 1000;
return series.filter(([ts,]) => parseTs(ts) >= cutoff);
}
function buildDatasets(data, metric) {
const datasets = [];
let idx = 0;
for (const client of Object.keys(data).sort()) {
const series = filterByRange(data[client]);
const points = series
.filter(([ts, m]) => m[metric] !== undefined)
.map(([ts, m]) => ({ x: ts.replace(" ", "T"), y: m[metric] }));
if (points.length === 0) continue;
const isCtrl = client.toLowerCase().includes("jerry");
datasets.push({
label: client + (isCtrl ? " (control)" : ""),
data: points,
borderColor: pickColor(client, idx),
backgroundColor: pickColor(client, idx) + "44",
borderWidth: isCtrl ? 2.5 : 1.5,
borderDash: isCtrl ? [6,3] : [],
pointRadius: 0,
tension: 0.15,
});
idx++;
}
return datasets;
}
// Compute per-client summary: latest value + hourly growth rate (last 4 snapshots) + projected hours to cap.
function buildSummary(data) {
const rows = [];
for (const client of Object.keys(data)) {
const series = data[client];
if (series.length === 0) continue;
const isCtrl = client.toLowerCase().includes("jerry");
const last = series[series.length-1];
const lastTs = last[0];
const m = last[1];
const lastPid = m.pid;
// Uptime = time since first row with the CURRENT PID (each restart gets
// a new PID, so this excludes pre-restart rows for the same character).
let firstTsForPid = lastTs;
for (const [ts, row] of series) {
if (row.pid === lastPid) { firstTsForPid = ts; break; }
}
const uptimeH = (parseTs(lastTs) - parseTs(firstTsForPid)) / 3600 / 1000;
// growth rate = slope from last min(4, length) snapshots
const tail = series.slice(Math.max(0, series.length - 4));
const rates = {};
if (tail.length >= 2) {
const t0 = parseTs(tail[0][0]);
const t1 = parseTs(tail[tail.length-1][0]);
const dtH = (t1 - t0) / 3600 / 1000;
if (dtH > 0) {
for (const key of METRICS) {
const v0 = tail[0][1][key];
const v1 = tail[tail.length-1][1][key];
if (v0 !== undefined && v1 !== undefined) rates[key] = (v1 - v0) / dtH;
}
}
}
// projected hours to 2GB cap
let hoursToCap = Infinity;
if (rates.mb !== undefined && rates.mb > 0 && m.mb !== undefined) {
hoursToCap = (CAP_MB - m.mb) / rates.mb;
}
rows.push({
client, isCtrl, lastTs, uptimeH, m, rates, hoursToCap
});
}
return rows;
}
function fmt(v, decimals=0) {
if (v === undefined || v === null) return "-";
if (!isFinite(v)) return "";
return v.toLocaleString(undefined, {maximumFractionDigits: decimals, minimumFractionDigits: decimals});
}
function rateClass(r) { if (!isFinite(r) || r === undefined) return ""; if (r === 0) return "grow-rate-zero"; return "grow-rate-pos"; }
function capClass(h) {
if (!isFinite(h)) return "ok";
if (h < 24) return "danger";
if (h < 72) return "warn";
return "ok";
}
function renderSummary(data) {
const rows = buildSummary(data);
rows.sort((a, b) => {
let av, bv;
if (sortKey === "client") { av = a.client; bv = b.client; }
else if (sortKey === "uptime"){ av = a.uptimeH; bv = b.uptimeH; }
else if (sortKey === "cap") { av = a.hoursToCap; bv = b.hoursToCap; }
else if (sortKey.endsWith("_rate")) {
const k = sortKey.replace("_rate","");
av = a.rates[k] ?? -1; bv = b.rates[k] ?? -1;
}
else { av = a.m[sortKey] ?? -1; bv = b.m[sortKey] ?? -1; }
if (typeof av === "string") return sortDir * av.localeCompare(bv);
return sortDir * (av - bv);
});
const colDefs = [
{key:"client", label:"Client"},
{key:"uptime", label:"Uptime (h)"},
{key:"mb", label:"MB"},
{key:"mb_rate", label:"MB/hr"},
{key:"cap", label:"hrs→2GB"},
{key:"palette", label:"Pal"},
{key:"palette_rate", label:"Pal/hr"},
{key:"renderSurf", label:"RSurf"},
{key:"renderSurf_rate", label:"RSurf/hr"},
{key:"renderSurfD3D", label:"RSD3D"},
{key:"renderSurfD3D_rate", label:"RSD3D/hr"},
{key:"csurface", label:"CSurf"},
{key:"csurface_rate", label:"CSurf/hr"},
{key:"imgtex", label:"ImgT"},
{key:"imgtex_rate", label:"ImgT/hr"},
];
let html = "<thead><tr>";
for (const c of colDefs) {
const arrow = (sortKey === c.key) ? (sortDir < 0 ? "" : "") : "";
html += `<th onclick="sortBy('${c.key}')">${c.label}${arrow}</th>`;
}
html += "</tr></thead><tbody>";
for (const r of rows) {
const cls = r.isCtrl ? "control" : capClass(r.hoursToCap);
html += `<tr class="${cls}">`;
html += `<td class="client">${r.client}${r.isCtrl?'<span class="pill pill-ctrl">control</span>':''}</td>`;
html += `<td>${fmt(r.uptimeH, 1)}</td>`;
html += `<td>${fmt(r.m.mb)}</td>`;
html += `<td class="${rateClass(r.rates.mb)}">${fmt(r.rates.mb, 1)}</td>`;
html += `<td class="cap">${fmt(r.hoursToCap, 0)}</td>`;
html += `<td>${fmt(r.m.palette)}</td>`;
html += `<td class="${rateClass(r.rates.palette)}">${fmt(r.rates.palette, 0)}</td>`;
html += `<td>${fmt(r.m.renderSurf)}</td>`;
html += `<td class="${rateClass(r.rates.renderSurf)}">${fmt(r.rates.renderSurf, 0)}</td>`;
html += `<td>${fmt(r.m.renderSurfD3D)}</td>`;
html += `<td class="${rateClass(r.rates.renderSurfD3D)}">${fmt(r.rates.renderSurfD3D, 0)}</td>`;
html += `<td>${fmt(r.m.csurface)}</td>`;
html += `<td class="${rateClass(r.rates.csurface)}">${fmt(r.rates.csurface, 0)}</td>`;
html += `<td>${fmt(r.m.imgtex)}</td>`;
html += `<td class="${rateClass(r.rates.imgtex)}">${fmt(r.rates.imgtex, 0)}</td>`;
html += `</tr>`;
}
html += "</tbody>";
document.getElementById("summary").innerHTML = html;
}
function sortBy(k) {
if (sortKey === k) sortDir *= -1;
else { sortKey = k; sortDir = (k === "client") ? 1 : -1; }
if (window._lastData) renderSummary(window._lastData);
}
function setRange(h) {
rangeHours = h;
if (window._lastData) {
for (const m of METRICS) {
charts[m].data.datasets = buildDatasets(window._lastData, m);
charts[m].update("none");
}
}
}
function makeChart(metric, canvas, data) {
const ctx = canvas.getContext("2d");
return new Chart(ctx, {
type: "line",
data: { datasets: buildDatasets(data, metric) },
options: {
animation: false,
responsive: true,
maintainAspectRatio: false,
scales: {
x: { type: "time",
time: { unit: "hour", tooltipFormat: "yyyy-MM-dd HH:mm" },
ticks: { color: "#999", maxRotation: 0 },
grid: { color: "#222" } },
y: { ticks: { color: "#999" }, grid: { color: "#222" }, beginAtZero: true }
},
plugins: {
legend: { labels: { color: "#ccc", boxWidth: 12, font:{size:10} }, position:"bottom" },
tooltip: { mode: "nearest" }
}
}
});
}
async function loadData() {
const status = document.getElementById("status");
status.textContent = "loading...";
try {
const res = await fetch("/data.json", { cache: "no-store" });
const data = await res.json();
window._lastData = data;
const grid = document.getElementById("charts");
const totalRows = Object.values(data).reduce((a,s)=>a+s.length,0);
const lastTs = Object.values(data).flatMap(s=>s.map(p=>p[0])).sort().slice(-1)[0] || "n/a";
status.textContent = `${Object.keys(data).length} clients, ${totalRows} datapoints, last @ ${lastTs}`;
renderSummary(data);
if (grid.children.length === 0) {
// build cards first time
for (const m of METRICS) {
const card = document.createElement("div");
card.className = "chart-card";
const h = document.createElement("h2");
h.textContent = METRIC_LABELS[m] || m;
const canvas = document.createElement("canvas");
canvas.id = "c_" + m;
card.append(h); card.append(canvas);
grid.append(card);
charts[m] = makeChart(m, canvas, data);
}
} else {
// update existing
for (const m of METRICS) {
charts[m].data.datasets = buildDatasets(data, m);
charts[m].update("none");
}
}
} catch (e) {
status.textContent = "error: " + e.message;
}
}
function toggleAutoRefresh() {
const btn = document.getElementById("autoBtn");
if (autoTimer) {
clearInterval(autoTimer);
autoTimer = null;
btn.textContent = "auto-refresh: off";
} else {
autoTimer = setInterval(loadData, 60000); // every 60s
btn.textContent = "auto-refresh: 60s";
}
}
loadData();
</script>
</body></html>
"""
HTML = HTML.replace("__METRICS__", json.dumps(METRICS))
HTML = HTML.replace("__METRIC_LABELS__", json.dumps(METRIC_LABELS))
REPORT_HTML = """<!doctype html>
<html><head>
<meta charset="utf-8">
<title>Leak-Hunt Final Report</title>
<script src="https://cdn.jsdelivr.net/npm/marked/marked.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/highlight.js@11.9.0/lib/core.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/highlight.js@11.9.0/lib/languages/c.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/highlight.js@11.9.0/lib/languages/cpp.min.js"></script>
<script src="https://cdn.jsdelivr.net/npm/highlight.js@11.9.0/lib/languages/x86asm.min.js"></script>
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/highlight.js@11.9.0/styles/github-dark.min.css">
<style>
body { font-family: -apple-system, system-ui, "Segoe UI", sans-serif; margin: 0; background: #0f1419; color: #e6e6e6; }
.nav { background: #1a1f29; border-bottom: 1px solid #2a3140; padding: 10px 24px; font-size: 13px; }
.nav a { color: #5da5da; text-decoration: none; margin-right: 16px; }
.nav a:hover { color: #8ec7ff; }
main { max-width: 980px; margin: 24px auto; padding: 0 32px 80px; line-height: 1.55; }
h1, h2, h3 { color: #fff; border-bottom: 1px solid #2a3140; padding-bottom: 6px; margin-top: 28px; }
h1 { font-size: 28px; }
h2 { font-size: 20px; }
h3 { font-size: 16px; border-bottom: none; }
h4 { font-size: 14px; border-bottom: none; }
p, li { color: #d8d8d8; }
a { color: #5da5da; }
code { background: #1a1f29; padding: 1px 5px; border-radius: 3px; font-size: 90%; color: #f1c40f; }
pre { background: #0c1014 !important; border: 1px solid #2a3140; border-radius: 5px; padding: 12px; overflow-x: auto; }
pre code { background: transparent; padding: 0; color: #d8d8d8; font-size: 12.5px; line-height: 1.45; }
table { border-collapse: collapse; margin: 12px 0; }
th, td { border: 1px solid #2a3140; padding: 5px 10px; text-align: left; font-size: 13px; }
th { background: #1a1f29; color: #ccc; }
tr:nth-child(even) td { background: #15191f; }
blockquote { border-left: 3px solid #5da5da; margin: 8px 0; padding: 4px 16px; color: #aaa; background: #1a1f29; }
hr { border: 0; border-top: 1px solid #2a3140; margin: 24px 0; }
</style>
</head><body>
<div class="nav">
<a href="/"> Dashboard</a>
<a href="/report">Final Report</a>
&nbsp;<span style="color:#666;">Asheron's Call Memory Leak Hunt</span>
</div>
<main id="content">Loading</main>
<script>
async function loadReport() {
const r = await fetch('/report.md');
if (!r.ok) { document.getElementById('content').innerHTML = '<p>Failed to load REPORT.md (HTTP '+r.status+')</p>'; return; }
const md = await r.text();
// Configure marked
marked.setOptions({
gfm: true,
breaks: false,
headerIds: true,
});
document.getElementById('content').innerHTML = marked.parse(md);
// Re-run highlight on code blocks
document.querySelectorAll('pre code').forEach(block => {
try { hljs.highlightElement(block); } catch(e) {}
});
}
hljs.registerLanguage('c', hljs.getLanguage ? undefined : null); // no-op safety
loadReport();
</script>
</body></html>
"""
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self, fmt, *args): # silence per-request logs
pass
def do_GET(self):
if self.path == "/" or self.path.startswith("/index"):
self._send(200, "text/html", HTML.encode("utf-8"))
elif self.path.startswith("/data.json"):
data = load_tsv()
payload = json.dumps(data).encode("utf-8")
self._send(200, "application/json", payload)
elif self.path.startswith("/report.md"):
try:
body = REPORT_MD.read_bytes()
self._send(200, "text/markdown; charset=utf-8", body)
except FileNotFoundError:
self._send(404, "text/plain", b"REPORT.md not found\n")
elif self.path == "/report" or self.path.startswith("/report/"):
self._send(200, "text/html", REPORT_HTML.encode("utf-8"))
else:
self._send(404, "text/plain", b"not found\n")
def _send(self, code, ctype, body):
self.send_response(code)
self.send_header("Content-Type", ctype)
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
self.wfile.write(body)
def main():
with socketserver.ThreadingTCPServer(("127.0.0.1", PORT), Handler) as srv:
srv.allow_reuse_address = True
print(f"Dashboard listening on http://localhost:{PORT}")
srv.serve_forever()
if __name__ == "__main__":
main()