From a564c4b78220458891fc31d623322acaa832800c Mon Sep 17 00:00:00 2001 From: Erik Date: Fri, 24 Jul 2026 15:49:24 +0200 Subject: [PATCH] perf(content): close prepared asset cutover --- AGENTS.md | 14 +- CLAUDE.md | 14 +- docs/ISSUES.md | 20 ++- docs/README.md | 33 +++-- docs/architecture/worldbuilder-inventory.md | 37 +++-- docs/plans/2026-04-11-roadmap.md | 18 ++- .../2026-07-24-modern-runtime-architecture.md | 27 ++-- ...-modern-runtime-slice-c-prepared-assets.md | 35 ++++- ...slice-c-prepared-asset-cutover-report.json | 134 ++++++++++++++++++ ...4-slice-c-prepared-asset-cutover-report.md | 123 ++++++++++++++++ src/AcDream.Content/DatCollectionAdapter.cs | 39 +++-- 11 files changed, 428 insertions(+), 66 deletions(-) create mode 100644 docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.json create mode 100644 docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.md diff --git a/AGENTS.md b/AGENTS.md index 621249df..bdbd81fa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -130,6 +130,18 @@ user-accepted. Favorite-spell press-time selection and right-click local examination are implemented; the current visual gate is its corrected modern scarab/prismatic formula, DAT component icons, and authored 310 x 400 extent. +**Active performance side track:** Slices A–C of +`docs/plans/2026-07-24-modern-runtime-architecture.md` completed 2026-07-24. +Production world-mesh streaming now consumes the validated machine-local +`acdream.pak` through `IPreparedAssetSource`; live DAT extraction remains +explicit bake/equivalence/UI-Studio tooling. Capped, uncapped, dense-Arwic, +lifetime, full-suite, and user-visual gates passed. Process allocation fell +41.8–61.9%, GC pause time fell 48.4–54.1%, uncapped CPU/GPU p99 improved +17.0%/20.6%, and invalid Setup-probe exceptions fell to zero. Slice D (typed +handles and unified residency accounting) is next; ECS/MP3 remains behind its +explicit user decision gate. Evidence: +`docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.md`. + **Structural prerequisite before new M4 subsystem work:** all eight behavior-preserving `GameWindow` decomposition slices and the automated closeout landed 2026-07-22. The class is now a 1,622-line native @@ -149,7 +161,7 @@ uncapped Release presentation is smooth, while physics, collision, and wire truth remain correct. See `docs/plans/2026-07-22-gamewindow-slice-8-composition-lifecycle.md` and `docs/architecture/code-structure.md`. **Carried:** #153, #116, remaining -R6 ownership cleanup, TS-50/TS-51/TS-53, Modern Pipeline MP1b+, and #225's +R6 ownership cleanup, TS-50/TS-51/TS-53, Modern Runtime Slice D+, and #225's lifestone/particle alpha visual gate. Start structural work at `memory/project_gamewindow_decomposition.md` and diff --git a/CLAUDE.md b/CLAUDE.md index c706d4a4..0ea9a8ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -128,6 +128,18 @@ user-accepted. Favorite-spell press-time selection and right-click local examination are implemented; the current visual gate is its corrected modern scarab/prismatic formula, DAT component icons, and authored 310 x 400 extent. +**Active performance side track:** Slices A–C of +`docs/plans/2026-07-24-modern-runtime-architecture.md` completed 2026-07-24. +Production world-mesh streaming now consumes the validated machine-local +`acdream.pak` through `IPreparedAssetSource`; live DAT extraction remains +explicit bake/equivalence/UI-Studio tooling. Capped, uncapped, dense-Arwic, +lifetime, full-suite, and user-visual gates passed. Process allocation fell +41.8–61.9%, GC pause time fell 48.4–54.1%, uncapped CPU/GPU p99 improved +17.0%/20.6%, and invalid Setup-probe exceptions fell to zero. Slice D (typed +handles and unified residency accounting) is next; ECS/MP3 remains behind its +explicit user decision gate. Evidence: +`docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.md`. + **Structural prerequisite before new M4 subsystem work:** all eight behavior-preserving `GameWindow` decomposition slices and the automated closeout landed 2026-07-22. The class is now a 1,622-line native @@ -147,7 +159,7 @@ uncapped Release presentation is smooth, while physics, collision, and wire truth remain correct. See `docs/plans/2026-07-22-gamewindow-slice-8-composition-lifecycle.md` and `docs/architecture/code-structure.md`. **Carried:** #153, #116, remaining -R6 ownership cleanup, TS-50/TS-51/TS-53, Modern Pipeline MP1b+, and #225's +R6 ownership cleanup, TS-50/TS-51/TS-53, Modern Runtime Slice D+, and #225's lifestone/particle alpha visual gate. Start structural work at `memory/project_gamewindow_decomposition.md` and diff --git a/docs/ISSUES.md b/docs/ISSUES.md index 2cc0ad39..e9fbc33c 100644 --- a/docs/ISSUES.md +++ b/docs/ISSUES.md @@ -24,7 +24,7 @@ What does NOT go here: - Every session: scan OPEN issues at start; promote/close anything we touched during the session before ending. - Promoting to a Phase: mark as `DONE (promoted to Phase X)` + commit SHA where the Phase entry landed. -## Current queue — 2026-07-23 +## Current queue — 2026-07-24 - **Structural handoff:** all eight `GameWindow` decomposition slices and the automated closeout are complete. The 1,622-line native shell has typed @@ -44,6 +44,9 @@ What does NOT go here: smooth and physics, collision, and wire state remain correct. - **Build hygiene:** `#228` records 17 clean-Release test-project warnings; production compilation and all tests pass. +- **Performance side track:** prepared-content Slices A–C are complete; + `#244` is closed and Slice D unified residency/accounting is next. Evidence: + [`research/2026-07-24-slice-c-prepared-asset-cutover-report.md`](research/2026-07-24-slice-c-prepared-asset-cutover-report.md). The [documentation map](README.md) defines how this tactical ledger relates to the milestone, roadmap, architecture, divergence register, research, and @@ -453,7 +456,7 @@ insert site ~1757). ## #244 — SequencerFactory probes Setup without a DID guard or catch; [dat-miss] logs under the database lock -**Status:** OPEN +**Status:** DONE — 2026-07-24, Modern Runtime Slice C **Severity:** MEDIUM **Filed:** 2026-07-24 **Component:** composition / content @@ -470,11 +473,14 @@ probes hit the dated TEMP `[dat-miss]` `Console.WriteLine` synchronous console I/O while holding `_databaseLock`, serializing all readers of that dat. -**Root cause / status:** 2026-07-24 audit review finding. Plan Slice C's -typed DID metadata removes the probe pattern; the Slice C amendment also -requires narrowing `ResolveActivation`'s blanket catch. The `[dat-miss]` -tripwire should be resolved (root-caused or demoted to a rate-limited -diagnostic) in the same change. +**Root cause / status:** Slice C's prepared TOC presence index now gates both +`SequencerFactory` and `ResolveActivation`, so GfxObj IDs do not enter the +Setup parser. Known malformed DAT failures are diagnosed once; cancellation +and unexpected failures propagate. The remaining `[dat-miss]` anomaly report +now captures its condition while holding the serialized DAT lock, performs +console I/O only after releasing it, and is limited to one report per database. +Connected capped, uncapped, and dense routes recorded zero invalid Setup-probe +exceptions. **Files:** `src/AcDream.App/Composition/LivePresentationComposition.cs:200-225`; `src/AcDream.Content/DatCollectionAdapter.cs:196-206`; diff --git a/docs/README.md b/docs/README.md index 58380370..a1dc787f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,31 +4,34 @@ This page is the entry point for project documentation. It distinguishes current sources of truth from implementation history so an old plan or issue banner cannot silently override the current program state. -## Current snapshot — 2026-07-21 +## Current snapshot — 2026-07-24 - **Milestone state:** M3, “Cast a spell,” landed 2026-07-21. M4, “Live in the - world,” is next. -- **Active program:** behavior-preserving `GameWindow` decomposition before - new M4 subsystem work. The reconciled ownership ledger and slice order are - in [`architecture/code-structure.md`](architecture/code-structure.md). - Selection/interaction Slice 1 is complete; live animation presentation is - next. + world,” is active. +- **Active gameplay program:** the pre-M4 world-interaction completion program + in + [`plans/2026-07-23-world-interaction-completion.md`](plans/2026-07-23-world-interaction-completion.md). + The eight-slice `GameWindow` decomposition and its connected visual matrix + are complete. +- **Active performance side track:** Slices A–C of + [`plans/2026-07-24-modern-runtime-architecture.md`](plans/2026-07-24-modern-runtime-architecture.md) + are complete. Production world-mesh streaming consumes the validated + prepared package; Slice D unified residency/accounting is next. See the + [Slice C physical cutover report](research/2026-07-24-slice-c-prepared-asset-cutover-report.md). - **Completed gates:** R6 locomotion/collision/projectile/teleport/radar and the final two-client portal-out/materialization observer comparison (`#218`). - **Separate visual verification:** the shared-alpha lifestone/particle result in `#225`; its connected resource-lifetime and performance routes pass. - **Carried behavior debt:** `#153` (far teleport onto an unstreamed edge), `#116` (two narrowed slide-response cases), and divergence rows TS-50/TS-51. -- **Deferred engineering tracks:** Modern Pipeline MP1b+ and Linux/headless - automation (Track LH). Neither is part of the active structural campaign. +- **Deferred engineering tracks:** render ECS/MP3 remains behind its explicit + user decision gate; Linux/headless automation (Track LH) remains future work. - **Divergence audit:** 178 active rows — IA 17, AD 37, AP 85, TS 34, and UN 5 — plus the retained retired TS-37 history note. -- **Latest automated baseline:** Release build succeeds with 17 known - test-project warnings (`#228`); 6,558 tests passed and 5 intentionally - skipped. The unattended connected R6 route - completed seven portal materializations, production input exercises, and a - graceful close. See - [`research/2026-07-20-connected-r6-soak.md`](research/2026-07-20-connected-r6-soak.md). +- **Latest automated baseline:** Release build succeeds; 8,076 tests pass and + five are intentionally skipped. The capped, uncapped, and dense-Arwic + physical prepared-asset routes all completed their portal/input/lifetime + assertions and disconnected gracefully. ## Sources of truth diff --git a/docs/architecture/worldbuilder-inventory.md b/docs/architecture/worldbuilder-inventory.md index 4680da46..31150a1b 100644 --- a/docs/architecture/worldbuilder-inventory.md +++ b/docs/architecture/worldbuilder-inventory.md @@ -35,9 +35,10 @@ acdream streams asynchronously while the WB editor's manager owned its own loadi loop. Do not reintroduce worker-side `RegisterCell` calls or shared pending cell collections. -`DatCollectionAdapter` bridges our `IDatCollection` to the `IDatReaderWriter` -interface WB's internals expect (O-D7 fallback; `ObjectMeshManager` has 26 -internal `_dats.*` call sites — above the 20-site inline-swap threshold). +`DatCollectionAdapter` bridges the sole `DatCollection` to Content's +`IDatReaderWriter`. Since MP1c, production `ObjectMeshManager` no longer reads +DAT; the adapter remains the bounded typed-object access seam for runtime +non-render content plus explicit bake/equivalence tooling. **MP1a (2026-07-05): CPU mesh-extraction half moved to `AcDream.Content`.** The GL-free portion of the former `ObjectMeshManager` — dat read → polygon @@ -82,14 +83,13 @@ behavior change, no divergence-register row. throw (preloads staged before a malformed-dat texture-decode exception survive, as they always did). The MP1b bake tool passes its own collector. -- **Stays in `src/AcDream.App/Rendering/Wb/`:** `ObjectMeshManager` (now a - thin wrapper owning the staged-queue/worker-pool/Dispose-quiesce lifecycle - and all GL upload — constructs one `MeshExtractor` and delegates every - former `Prepare*` call site to it), `ObjectRenderData`/`ObjectRenderBatch` +- **Stays in `src/AcDream.App/Rendering/Wb/`:** `ObjectMeshManager` (the + staged-queue/worker-pool/Dispose-quiesce lifecycle and all GL upload; + production workers now consume `IPreparedAssetSource`), + `ObjectRenderData`/`ObjectRenderBatch` (hold a GL `TextureAtlasManager` field), `TextureAtlasManager`, - `DatCollectionAdapter` (concrete `DatCollection`-backed implementation of - the now-Content-namespaced `IDatReaderWriter`), `GeometryUtils` (used only - by App-side raycasting, not by extraction), `AcSurfaceMetadata`/ + `GeometryUtils` (used only by App-side raycasting, not by extraction), + `AcSurfaceMetadata`/ `AcSurfaceMetadataTable` (not on the extraction path), `Building.cs` (explicitly out of scope). - `AcDream.Core` is untouched; `AcDream.Content` references `AcDream.Core` @@ -114,6 +114,23 @@ aliasing and fails on any collision. This is an acdream resource-ownership seam, not a second DAT interpreter; `DatCollection` and `MeshExtractor` remain the only reader/extractor path. +**MP1c production prepared-asset cutover (2026-07-24).** Production +world-mesh workers no longer invoke `MeshExtractor` or rebuild Setup, GfxObj, +EnvCell, Surface, palette, and texture graphs during portals. The validated +machine-local `acdream.pak` is opened through Content's +`IPreparedAssetSource`; typed GfxObj and EnvCell requests deserialize immutable +`ObjectMeshData` while retaining the existing App worker, staging, render-thread +upload, cache, ownership, and shutdown contracts. The format-1/bake-tool-3 +payload persists exact batch translucency so App does not reconstruct a +`GfxObjMesh` for metadata. Setup activation uses the package TOC as an explicit +type-presence index before reading valid Setup records through the bounded DAT +cache. `DatPreparedAssetSource` and `MeshExtractor` remain explicit +bake/equivalence/UI-Studio tools, not a production fallback. Portal → HighRes +→ Language → Cell lookup precedence is encoded directly in +`DatCollectionAdapter.TryResolvePreferred`. The connected physical and +installed-DAT gates are recorded in +`docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.md`. + **Retail VFX hook compatibility seam (2026-07-14).** Chorizite.DatReaderWriter 2.1.7 models `CreateBlockingParticleHook` as the common hook header only, while retail inherits the complete `CreateParticleHook` payload. The narrow readers in diff --git a/docs/plans/2026-04-11-roadmap.md b/docs/plans/2026-04-11-roadmap.md index 243d7d09..36f59b76 100644 --- a/docs/plans/2026-04-11-roadmap.md +++ b/docs/plans/2026-04-11-roadmap.md @@ -1562,9 +1562,10 @@ port in any phase — no separate listing here. > remains DEFERRED**: ECS is the *throughput* lever (raise avg FPS ~230→300), a > DIFFERENT goal from steadiness — revisit ONLY if the user wants a higher raw FPS > number and accepts the rewrite risk. Rust rejected. Parked (resume from the table -> below): MP1b pak/bake is now complete and the corrected physical-local -> baselines have landed, so MP1c is active. The remaining parked items are -> MP-Alloc hard sites and MP1c. MP2 has shipped. Full rationale: +> below): MP1b pak/bake and MP1c production cutover are now complete. The +> remaining parked items are MP-Alloc hard sites; typed residency and +> cost-budgeted streaming continue as Slices D/E in the modern-runtime plan. +> MP2 has shipped. Full rationale: > `project_mp_track_findings.md`. > The 2026-07-18 portal/resource-lifetime repair does not resume MP's ECS, pak, > or throughput redesign. It is corrective ownership/reclamation work for a @@ -1588,10 +1589,15 @@ port in any phase — no separate listing here. > portal windows, observer-effect measurement, binary provenance, and > contention stacks. The physical runs confirm fast steady rendering but also > preserve repeated-portal memory plateau failures, ~200 MiB allocation frames, -> and thousands of invalid typed-Setup probes as MP1c's before-state. Slice C -> is unblocked and active. Slice B EnvCell dedup is complete; its +> and thousands of invalid typed-Setup probes as MP1c's before-state. Slice B +> EnvCell dedup is complete; its > [full-bake report](../research/2026-07-24-slice-b-full-bake-report.md) > records the 42.6× result and corrected geometry-identity collision. +> **Slice C completed 2026-07-24:** production streaming consumes the validated +> bake-tool-3 package, all physical connected and user-visual gates passed, +> process allocation fell 41.8–61.9%, GC pause time fell 48.4–54.1%, and +> invalid typed-Setup exceptions fell to zero. [Cutover +> report](../research/2026-07-24-slice-c-prepared-asset-cutover-report.md). **Spec:** `docs/superpowers/specs/2026-07-05-modern-pipeline-design.md` (the umbrella design — read it first). **Goal:** smoothness first (no frame over @@ -1616,7 +1622,7 @@ hitch and follows. | MP1b | Pak format + `acdream-bake` CLI + mmap `PakReader` + equivalence/full-scale gate | ✅ **SHIPPED 2026-07-24** — 729,888 EnvCell keys → 17,117 unique geometries + 712,771 aliases (42.6×), 751,141 total keys / 38,370 physical blobs, zero failures, 28,192.4 MiB, 81.4 s validated atomic publish; full-DAT gate also corrected a real collision in WB's legacy geometry hash. [Report](../research/2026-07-24-slice-b-full-bake-report.md) | | **MP-Alloc (safe batch)** | Reuse per-frame buffers: anim pose, particle draw-list, interior partition, animatedIds/drawableCells sets | ✅ SHIPPED + USER-GATED 2026-07-05 — dense-town frame-time spikes 20–87ms → 6–10ms, alloc spikes (30–75MB single-frame) eliminated, gen2 GC 5–11/window → ~0; user confirms FPS steady. 4 commits `b8c05e2b`..`91afea24` | | MP-Alloc (hard sites) | EnvCell settled-camera rebuild gate + physics `Transition` pooling — the residual steady ~1.6MB/frame | ⚪ OPTIONAL follow-up — would lower the median too, but the wild-swing complaint is already resolved; each needs its own careful gate (batch correctness / physics faithfulness) | -| MP1c | Streaming cutover to pak + hitch gate | **ACTIVE 2026-07-24** — physical-local Slice A gate committed; baseline: uncapped CPU p50/p95/p99 1.520/5.694/7.825 ms, GPU 0.897/3.239/3.406 ms, portal allocation max 212.7 MiB, repeated-Caul working-set growth 295.1 MiB. Slice B package foundation is complete | +| MP1c | Streaming cutover to pak + hitch gate | ✅ **SHIPPED 2026-07-24** — typed package-only production source, explicit Setup probes, non-allocating Portal-first lookup, exact translucency metadata, and ordered teardown; capped/uncapped/dense physical routes + user visual gate pass. Uncapped CPU p99 7.825→6.496 ms, GPU p99 3.406→2.706 ms, portal frame max 202.9→39.9 MiB, process allocation −41.8%, zero invalid Setup probes. [Report](../research/2026-07-24-slice-c-prepared-asset-cutover-report.md) | | MP2 | Retail particle distance/cell-view degradation: DAT-authored range, exact infinite/finite off-view update branches, emitter-first render scan, user-requested 2× default range | ✅ SHIPPED 2026-07-17 — Aerlinthe hotspot `0x32000223` resolves to retail 64m and defaults to 128m; landscape/entity streaming distance unchanged | | MP3 | Arch ECS render world + delta submission (the 300-FPS lever) | ⚪ — note: does NOT fix the steady-state GC churn (that's MP-Alloc); MP3 is the draw-submission throughput lever | | MP4 | Zero-alloc frame loop + flat physics data (residual, post-MP-Alloc) | ⚪ hard-queued behind M1.5 #137 | diff --git a/docs/plans/2026-07-24-modern-runtime-architecture.md b/docs/plans/2026-07-24-modern-runtime-architecture.md index ad373040..27419235 100644 --- a/docs/plans/2026-07-24-modern-runtime-architecture.md +++ b/docs/plans/2026-07-24-modern-runtime-architecture.md @@ -2,8 +2,7 @@ **Date:** 2026-07-24 -**Status:** Slice A complete on the physical reference display; Slice B -complete. Slice C is unblocked and active. +**Status:** Slices A–C complete. Slice D is next and not yet started. **Scope:** Reconcile and sequence the existing Modern Pipeline (`MP`) and Linux/headless (`LH`) tracks using the 2026-07-24 connected performance audit. @@ -631,6 +630,15 @@ collision in WorldBuilder's legacy 31× geometry hash. Evidence: single-frame allocation and p99 materially improve from the 2026-07-24 baseline; connected nine-stop route and screenshots pass. +**Completed 2026-07-24:** production streaming now consumes the validated +format-1/bake-tool-3 package through `IPreparedAssetSource`, with no implicit +live-DAT mesh fallback. Capped, uncapped, and dense-Arwic physical routes, +installed-DAT equivalence, lifecycle assertions, graceful teardown, the full +Release suite, and the user's visual gate passed. Process allocation fell +41.8–61.9%, GC pause time fell 48.4–54.1%, uncapped CPU/GPU p99 improved +17.0%/20.6%, and invalid typed-Setup exceptions fell to zero. Evidence: +[`../research/2026-07-24-slice-c-prepared-asset-cutover-report.md`](../research/2026-07-24-slice-c-prepared-asset-cutover-report.md). + **Amendment (2026-07-24 review).** Binding contracts for this slice: - **Precedence contract.** The LINQ being replaced is load-bearing content @@ -1129,13 +1137,14 @@ an improvement" carries the same evidence burden as a regression claim. ## 14. Current execution point -Slice A's tooling and authoritative physical-local evidence are complete, and -Slice B's prepared-content bake is complete. Slice C is active. Its comparison -baseline is the committed physical-local capture at -`49f3a48ee63b82d00731b0f91497930925279c86`: uncapped CPU p50/p95/p99 -1.520/5.694/7.825 ms, GPU p50/p95/p99 0.897/3.239/3.406 ms, portal allocation -max 212.7 MiB, process allocation mean 50.0 MB/s, and a repeated-Caul working -set plateau failure of 295.1 MiB. +Slices A–C are complete. Slice C replaced production live-DAT world-mesh +extraction with the validated prepared source and passed all connected, +performance, lifetime, automated, and user-visual gates. Its physical +uncapped result is CPU p50/p95/p99 1.409/4.909/6.496 ms, GPU +0.832/2.510/2.706 ms, a 39.9 MiB largest frame allocation, zero invalid Setup +probe exceptions, and a passing warm repeated-Caul memory assertion. Slice D +is the next execution point; it unifies accounting and policy around the +specialized caches that remain, rather than replacing their working storage. The intended order is therefore: diff --git a/docs/plans/2026-07-24-modern-runtime-slice-c-prepared-assets.md b/docs/plans/2026-07-24-modern-runtime-slice-c-prepared-assets.md index bd30cac1..5125f2fd 100644 --- a/docs/plans/2026-07-24-modern-runtime-slice-c-prepared-assets.md +++ b/docs/plans/2026-07-24-modern-runtime-slice-c-prepared-assets.md @@ -2,15 +2,18 @@ **Date:** 2026-07-24 -**Status:** active — C1 and C2 landed; C3 implemented, C4 connected gate next +**Status:** complete — automated, connected, performance, and user visual +gates passed 2026-07-24 **Checkpoint ledger:** - C1 typed prepared-source boundary — `c42f93b3` - C2 production renderer injection — `f05afc07` - C3 activation probes and lookup precedence — `230a7df4`; automated gate - green, connected exception-count gate remains in C4 -- C4 connected equivalence and performance closeout — pending + green +- C4 prepared-source diagnostics — `b1ad4b7c` +- C4 connected equivalence and performance closeout — + [`../research/2026-07-24-slice-c-prepared-asset-cutover-report.md`](../research/2026-07-24-slice-c-prepared-asset-cutover-report.md) **Parent:** [`2026-07-24-modern-runtime-architecture.md`](2026-07-24-modern-runtime-architecture.md), Slice C / MP1c @@ -300,6 +303,32 @@ If pixels or behavior differ, the slice is not complete even when performance improves. If timing improves but heap reservation still fails, keep the failure open and attribute it before considering Slice C closed. +### C4 result — completed 2026-07-24 + +The format-1/bake-tool-3 production package validated with 751,141 keys and +the exact installed DAT iterations. Installed-DAT seam/equivalence tests, all +three connected physical-display routes, canonical lifetime assertions, and +the complete Release suite passed. The user accepted the prepared path's +geometry, materials, textures, and world presentation. + +Against the committed Slice A physical baseline: + +- capped CPU p95/p99 improved 15.8%/16.9%, GPU p95/p99 17.4%/20.0%; +- uncapped CPU p95/p99 improved 13.8%/17.0%, GPU p95/p99 22.5%/20.6%; +- dense-Arwic CPU/GPU p99 improved 12.0%/16.8%; +- process allocation fell 54.9% capped, 41.8% uncapped, and 61.9% dense; +- GC pause time fell 52.8%, 48.4%, and 54.1%; +- the invalid typed-Setup exception count fell to zero in every route; +- the largest frame-thread allocation fell from roughly 203 MiB to 46.8 MiB + capped and 39.9 MiB uncapped; +- both warm repeated-Caul memory comparisons passed the unchanged assertion. + +Windows reports clean memory-mapped package pages in working set, so total +working set is higher even though private memory and ownership remain bounded. +That is file cache, not retained world state. Exact measurements and the +artifact policy are recorded in the +[Slice C report](../research/2026-07-24-slice-c-prepared-asset-cutover-report.md). + ## 4. Review checklist per checkpoint Because this task is running without delegated reviewers, the primary engineer diff --git a/docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.json b/docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.json new file mode 100644 index 00000000..31a05108 --- /dev/null +++ b/docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.json @@ -0,0 +1,134 @@ +{ + "schemaVersion": 1, + "capturedUtcDate": "2026-07-24", + "binaryCommit": "b1ad4b7c0a0a519ab36d707baba74890634e6028", + "result": "pass", + "visualGate": { + "status": "user-accepted", + "date": "2026-07-24", + "note": "Geometry, materials, textures, and world presentation accepted. Raw cross-run pixel differences were attributable to world lighting and post-turn camera state; no comparison tolerance was loosened." + }, + "pak": { + "formatVersion": 1, + "bakeToolVersion": 3, + "datIterations": { + "portal": 2072, + "cell": 982, + "highRes": 497, + "language": 994 + }, + "keys": { + "gfxObj": 15318, + "setup": 5935, + "envCell": 729888, + "total": 751141 + }, + "bytes": 29562150456, + "gibibytes": 27.53, + "validation": "pass" + }, + "runs": { + "capped": { + "baselineStem": "connected-r6-soak-20260724-141608", + "candidateStem": "connected-r6-soak-20260724-151701", + "candidateFrames": 111997, + "candidateRouteSeconds": 503.8, + "cpuMicroseconds": { + "baseline": { "p50": 4287, "p95": 6972, "p99": 9404 }, + "candidate": { "p50": 4247, "p95": 5873, "p99": 7814 } + }, + "gpuMicroseconds": { + "baseline": { "p50": 1702, "p95": 3261, "p99": 3847 }, + "candidate": { "p50": 1836, "p95": 2694, "p99": 3078 } + }, + "frameAllocationBytes": { + "baselineMaximum": 213380600, + "candidateMaximum": 49066536 + }, + "runtime": { + "baselineAllocatedBytes": 22611515504, + "candidateAllocatedBytes": 10207585632, + "baselineGcPauseSeconds": 2.9, + "candidateGcPauseSeconds": 1.368, + "baselineGcCollections": { "gen0": 191, "gen1": 159, "gen2": 54 }, + "candidateGcCollections": { "gen0": 82, "gen1": 84, "gen2": 26 }, + "baselineInvalidSetupProbeExceptions": 4851, + "candidateInvalidSetupProbeExceptions": 0 + }, + "warmCaulPlateauMibibytes": { + "workingSetGrowth": 80.6, + "privateGrowth": 53.8 + }, + "result": "pass" + }, + "uncapped": { + "baselineStem": "connected-r6-soak-20260724-142604", + "candidateStem": "connected-r6-soak-20260724-153039", + "candidateFrames": 252334, + "candidateRouteSeconds": 497.893, + "cpuMicroseconds": { + "baseline": { "p50": 1520, "p95": 5694, "p99": 7825 }, + "candidate": { "p50": 1409, "p95": 4909, "p99": 6496 } + }, + "gpuMicroseconds": { + "baseline": { "p50": 897, "p95": 3239, "p99": 3406 }, + "candidate": { "p50": 832, "p95": 2510, "p99": 2706 } + }, + "frameAllocationBytes": { + "baselineMaximum": 212712648, + "candidateMaximum": 41885648 + }, + "runtime": { + "baselineAllocatedBytes": 25308730712, + "candidateAllocatedBytes": 14738650240, + "baselineGcPauseSeconds": 2.567, + "candidateGcPauseSeconds": 1.325, + "baselineGcCollections": { "gen0": 278, "gen1": 143, "gen2": 58 }, + "candidateGcCollections": { "gen0": 188, "gen1": 83, "gen2": 23 }, + "baselineInvalidSetupProbeExceptions": 4659, + "candidateInvalidSetupProbeExceptions": 0 + }, + "warmCaulPlateauMibibytes": { + "workingSetGrowth": 232.0, + "privateGrowth": 183.7 + }, + "result": "pass" + }, + "denseArwic": { + "baselineStem": "connected-dense-town-20260724-143503", + "candidateStem": "connected-dense-town-20260724-153930", + "candidateFrames": 16732, + "candidateRouteSeconds": 57.394, + "cpuMicroseconds": { + "baseline": { "p50": 3672, "p95": 6561, "p99": 14121 }, + "candidate": { "p50": 3487, "p95": 6044, "p99": 12432 } + }, + "gpuMicroseconds": { + "baseline": { "p50": 2476, "p95": 3284, "p99": 3752 }, + "candidate": { "p50": 2126, "p95": 2768, "p99": 3122 } + }, + "frameAllocationBytes": { + "baselineMaximum": 29875456, + "candidateMaximum": 28200384 + }, + "runtime": { + "baselineAllocatedBytes": 5602582864, + "candidateAllocatedBytes": 2136709504, + "baselineGcPauseSeconds": 0.623, + "candidateGcPauseSeconds": 0.286, + "baselineGcCollections": { "gen0": 53, "gen1": 34, "gen2": 14 }, + "candidateGcCollections": { "gen0": 19, "gen1": 19, "gen2": 7 }, + "baselineInvalidSetupProbeExceptions": 331, + "candidateInvalidSetupProbeExceptions": 0 + }, + "result": "pass" + } + }, + "focusedValidation": { + "preparedSourceSerializerAndEquivalence": { "passed": 20, "failed": 0 }, + "fullBakeDeterminismAndArtifact": { "passed": 2, "failed": 0 }, + "releaseBuild": "pass", + "completeReleaseSuite": { "passed": 8076, "skipped": 5, "failed": 0 } + }, + "artifactPolicy": "The validated full pak is an untracked machine-local production input beside the DAT files. Raw routes, counters, screenshots, and logs remain untracked; this report is path-free committed evidence." +} diff --git a/docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.md b/docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.md new file mode 100644 index 00000000..9f612eaa --- /dev/null +++ b/docs/research/2026-07-24-slice-c-prepared-asset-cutover-report.md @@ -0,0 +1,123 @@ +# Slice C prepared-asset cutover report + +**Date:** 2026-07-24 + +**Measured binary:** `b1ad4b7c0a0a519ab36d707baba74890634e6028` + +**Result:** pass + +Slice C moves production world-mesh preparation out of the live client. +`acdream-bake` converts installed retail DAT geometry and material metadata +into an immutable `acdream.pak`; production streaming now reads those prepared +records without reconstructing the same DAT object graphs during portals. +DAT remains authoritative for simulation, UI, animation, dynamic appearance, +physics, audio, and effects. + +The user's connected screenshot review passed on 2026-07-24. Raw screenshot +differences against Slice A were caused by non-identical world lighting and +post-turn camera state; the user confirmed the prepared path's geometry, +materials, textures, and world presentation look correct. No tolerance was +widened and no world pixels were masked to obtain that result. + +## Production package + +| Measure | Result | +|---|---:| +| Format / bake-tool version | 1 / 3 | +| GfxObj keys | 15,318 | +| Setup keys | 5,935 | +| EnvCell keys | 729,888 | +| Total TOC keys | 751,141 | +| Portal / Cell / HighRes / Language iteration | 2072 / 982 / 497 / 994 | +| Artifact size | 29,562,150,456 bytes (27.53 GiB) | +| Header, TOC, ranges, aliases, CRCs, and iterations | pass | + +Bake-tool version 3 persists each texture batch's exact translucency +classification. That lets the render thread consume the prepared record +directly instead of rebuilding a `GfxObjMesh` solely to recover material +metadata. The validated package is a required machine-local production input +beside the DAT files; it is not a repository artifact. + +Focused installed-DAT validation passed 20 prepared-source, serializer, and +pak-equivalence tests plus two full-bake determinism/artifact tests. The +complete Release suite passed 8,076 tests with five skips and zero failures. + +## Connected physical-display result + +All three runs used the committed reference AMD display configuration, a +source-matched Release binary, the same deterministic routes as Slice A, and +normal retained gameplay UI. Every destination materialized, movement/jump/ +combat exercises passed, and every client disconnected gracefully. + +| Capture | CPU p50 / p95 / p99 | GPU p50 / p95 / p99 | Frame allocation max | +|---|---:|---:|---:| +| Capped Slice A | 4.287 / 6.972 / 9.404 ms | 1.702 / 3.261 / 3.847 ms | 203.5 MiB | +| Capped Slice C | 4.247 / 5.873 / 7.814 ms | 1.836 / 2.694 / 3.078 ms | 46.8 MiB | +| Uncapped Slice A | 1.520 / 5.694 / 7.825 ms | 0.897 / 3.239 / 3.406 ms | 202.9 MiB | +| Uncapped Slice C | 1.409 / 4.909 / 6.496 ms | 0.832 / 2.510 / 2.706 ms | 39.9 MiB | +| Dense Arwic Slice A | 3.672 / 6.561 / 14.121 ms | 2.476 / 3.284 / 3.752 ms | 28.5 MiB | +| Dense Arwic Slice C | 3.487 / 6.044 / 12.432 ms | 2.126 / 2.768 / 3.122 ms | 26.9 MiB | + +The capped path improved CPU p95/p99 by 15.8%/16.9% and GPU p95/p99 by +17.4%/20.0%. The uncapped path improved CPU p95/p99 by 13.8%/17.0% and GPU +p95/p99 by 22.5%/20.6%. Dense Arwic improved CPU p99 by 12.0% and GPU p99 by +16.8%. Capped GPU p50 rose by 0.134 ms, but every capped GPU tail percentile +improved; neither the uncapped nor dense captures repeated that median cost. + +## Allocation, GC, and probe result + +| Capture | Process allocation | GC pause | Gen0 / Gen1 / Gen2 | Invalid Setup probe exceptions | +|---|---:|---:|---:|---:| +| Capped Slice A | 22.61 GB | 2.900 s | 191 / 159 / 54 | 4,851 | +| Capped Slice C | 10.21 GB | 1.368 s | 82 / 84 / 26 | 0 | +| Uncapped Slice A | 25.31 GB | 2.567 s | 278 / 143 / 58 | 4,659 | +| Uncapped Slice C | 14.74 GB | 1.325 s | 188 / 83 / 23 | 0 | +| Dense Slice A | 5.60 GB | 0.623 s | 53 / 34 / 14 | 331 | +| Dense Slice C | 2.14 GB | 0.286 s | 19 / 19 / 7 | 0 | + +Process allocation fell 54.9% capped, 41.8% uncapped, and 61.9% in dense +Arwic. GC pause time fell 52.8%, 48.4%, and 54.1%, respectively. The explicit +prepared Setup presence probe eliminated the normal-control-flow +`ArgumentOutOfRangeException` storm without swallowing unexpected failures. + +The largest frame-thread allocation fell 77.0% capped and 80.3% uncapped. +This slice therefore removed most of the portal decode spike even though the +2026-07-24 audit correctly assigned the remaining publication/retirement work +to cost-budgeted streaming Slice E. + +## Residency interpretation + +Both repeated-Caul routes passed the existing warm-return/plateau memory +assertion without changing its threshold: + +- capped Caul return to plateau: +80.6 MiB working set, +53.8 MiB private; +- uncapped Caul return to plateau: +232.0 MiB working set, +183.7 MiB private. + +Handles and threads also returned to their established bands. First-to-final +working set is higher than Slice A because Windows includes clean pages from +the 27.53 GiB memory-mapped package in process working set. Those pages are +discardable file cache, not private heap or retained world ownership. Private +memory remained bounded across warm revisits, while entity, animation, +particle, mesh-upload, teardown, and ownership invariants passed. + +This is a correctness-first package, not the final disk/residency format. +Deduplication and compression may reduce its footprint later, but they were +deliberately excluded from the byte/field-equivalence cutover. + +## Architecture closeout + +- Production `ObjectMeshManager` reads only `IPreparedAssetSource`; it has no + live-DAT extraction fallback. +- `DatPreparedAssetSource` remains explicit tooling/equivalence infrastructure. +- GfxObj, Setup, and EnvCell identities are typed and validated. +- Missing, corrupt, canceled, and loaded reads remain distinct. +- Portal DAT wins the non-allocating preferred-ID lookup. +- Worker preparation and render-thread GL upload ownership remain unchanged. +- Mesh workers stop before the package is unmapped during shutdown. +- Prepared-source probe/read/result counters are exposed in the existing + render resource diagnostic record. + +Machine-readable, path-free evidence is in +[`2026-07-24-slice-c-prepared-asset-cutover-report.json`](2026-07-24-slice-c-prepared-asset-cutover-report.json). +Raw frame histories, counters, reports, screenshots, and logs remain local +untracked diagnostic artifacts. diff --git a/src/AcDream.Content/DatCollectionAdapter.cs b/src/AcDream.Content/DatCollectionAdapter.cs index 3b058955..e77bc1a3 100644 --- a/src/AcDream.Content/DatCollectionAdapter.cs +++ b/src/AcDream.Content/DatCollectionAdapter.cs @@ -5,14 +5,15 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; +using System.Threading; namespace AcDream.Content; /// /// THE → adapter, -/// shared by all three consumers: ObjectMeshManager (AcDream.App, via -/// WbMeshAdapter), acdream-bake (AcDream.Bake.BakeRunner), and the pak -/// equivalence suite (AcDream.Content.Tests). MP1b's review found the +/// shared by runtime DAT-backed gameplay/content access, +/// acdream-bake (AcDream.Bake.BakeRunner), and the pak equivalence suite +/// (AcDream.Content.Tests). MP1b's review found the /// original App-internal copy had silently forked into three near-identical /// versions — drift between them is exactly what the live-vs-pak equivalence /// suite CANNOT detect (both sides would drift together only if they share @@ -218,6 +219,7 @@ public sealed class DatDatabaseWrapper : IDatDatabase { // cache itself documents why estimated bytes are not a hard heap bound. private readonly BoundedDatObjectCache _cache = new(); private readonly object _databaseLock = new(); + private int _malformedRecordDiagnosticEmitted; public DatDatabaseWrapper(DatDatabase db) { ArgumentNullException.ThrowIfNull(db); @@ -241,6 +243,7 @@ public sealed class DatDatabaseWrapper : IDatDatabase { return true; } + bool existingRecordFailed; lock (_databaseLock) { // A different reader may have populated the cache while this // caller waited for the serialized DatDatabase read path. @@ -253,17 +256,25 @@ public sealed class DatDatabaseWrapper : IDatDatabase { return true; } - // TEMP diagnostic (dat-race investigation 2026-06-09, strip with fix): - // a miss for an id whose BTree entry EXISTS is always an anomaly — - // either Unpack returned false or the lookup flickered transiently. - // Legit not-found probes (e.g. Portal→HighRes fallback) stay silent. - // Kept through the MP1b unification — the tripwire now covers the - // bake tool and the equivalence suite too, not just the client. - if (_db.Tree.TryGetFile(fileId, out _)) { - Console.WriteLine( - $"[dat-miss] {typeof(T).Name} 0x{fileId:X8} entry EXISTS but TryGet failed " + - $"(thread={Environment.CurrentManagedThreadId})"); - } + // A miss for an id whose BTree entry exists is a typed-access + // anomaly (wrong requested type, malformed content, or an unpacker + // failure). Capture only the fact while serialized DAT access is + // held; diagnostics must never perform console I/O under this + // shared lock. + existingRecordFailed = _db.Tree.TryGetFile(fileId, out _); + } + + // One actionable report per database keeps a repeatedly requested + // malformed record from becoming another exception/logging-style hot + // path. Missing keys stay silent. + if (existingRecordFailed + && Interlocked.Exchange( + ref _malformedRecordDiagnosticEmitted, + 1) == 0) + { + Console.Error.WriteLine( + $"[dat-miss] {typeof(T).Name} 0x{fileId:X8} entry EXISTS but TryGet failed " + + $"(thread={Environment.CurrentManagedThreadId}; further reports suppressed)"); } return false;