From 01f4791e958f474e79254d92ef48e0217ae8d308 Mon Sep 17 00:00:00 2001 From: Erik Date: Sun, 2 Aug 2026 20:53:11 +0200 Subject: [PATCH] fix(streaming): stop replaying committed recenter retirements Root cause: pending-only live projection buckets were misclassified as landblock presentation owners during origin recentering. That manufactured a second full cleanup receipt for a generation whose first receipt was still advancing; the duplicate guard threw and the broad retry path replayed the already-committed detach 243 times. Keep pending live projections through the spatial identity map without issuing another receipt, and fail fast when a receipt-ledger invariant occurs after detachment. Evidence: docs/research/2026-08-02-collision-throughput-handoff/p1-retirement-receipt-loop.md. Release suite, lifecycle gate, and nine-stop soak pass. --- .../retail-divergence-register.md | 8 ++ .../implementer-progress.md | 33 ++++++ .../p1-retirement-receipt-loop.md | 95 ++++++++++++++++ src/AcDream.App/Streaming/GpuWorldState.cs | 10 +- .../LandblockPresentationPipeline.cs | 21 +++- .../Streaming/StreamingController.cs | 8 ++ .../Streaming/GpuWorldStateVisibilityTests.cs | 9 +- .../LandblockPresentationPipelineTests.cs | 106 ++++++++++++++++++ .../LandblockRetirementCoordinatorTests.cs | 28 +++++ 9 files changed, 309 insertions(+), 9 deletions(-) create mode 100644 docs/research/2026-08-02-collision-throughput-handoff/p1-retirement-receipt-loop.md diff --git a/docs/architecture/retail-divergence-register.md b/docs/architecture/retail-divergence-register.md index 2b8dcaf7..36e79f4e 100644 --- a/docs/architecture/retail-divergence-register.md +++ b/docs/architecture/retail-divergence-register.md @@ -80,6 +80,14 @@ loader branch there. Slice 4B must preserve the flag while mapping successful deferred placement to exact-cell, generation-scoped asynchronous admission; the presence of the flag in the immutable request is not claimed as exactness. +AD-2 retirement-receipt refinement (2026-08-02): a pending-only live +projection bucket survives the atomic origin swap but is not a landblock +presentation generation and emits no second full cleanup receipt. A genuine +receipt-ledger invariant after spatial detachment is a committed terminal +failure, never resumable detach work. This preserves the existing adaptation: +one exact asynchronous cleanup owner for each synchronously destroyed retail +landblock, while logical live objects survive streaming residence changes. + AP-1/AD-1 checkpoint (placement Slice 4B2 checkpoint 1, 2026-07-31): Runtime now owns the exact accepted placement/lost-cell transaction, atomic body/contact/cell/ shadow/workset commit, adjusted retained frame, authored mover preparation, diff --git a/docs/research/2026-08-02-collision-throughput-handoff/implementer-progress.md b/docs/research/2026-08-02-collision-throughput-handoff/implementer-progress.md index 79d0499a..87061cd1 100644 --- a/docs/research/2026-08-02-collision-throughput-handoff/implementer-progress.md +++ b/docs/research/2026-08-02-collision-throughput-handoff/implementer-progress.md @@ -4500,6 +4500,39 @@ Register row AD-61 records the "overwritten, not deleted" truth. passing signature (142539/161138/164432 class). 174811 interference attribution confirmed. Round complete; nothing staged. +## P1 — origin-recenter retirement-receipt exception loop + +Root cause pinned in +`p1-retirement-receipt-loop.md`. An already-detached landblock can retain +live projections in `GpuWorldState._pendingByLandblock` while its one exact +full-cleanup ticket advances. The recenter swap incorrectly promoted that +pending-only spatial bucket into a second full presentation receipt. The +coordinator rejected the duplicate correctly, but the controller's broad +resume catch replayed the already-committed detach 243 times in the captured +feel-test session. + +Implemented: + +- pending-only live buckets are retained through `_projectionLocations` but + no longer manufacture a landblock retirement receipt; +- loaded/pending-render/pending-near/tier/bounds owners still receive exact + receipts; +- a genuine receipt-ledger invariant after the spatial commit is surfaced as + a committed `StreamingMutationException` and cannot enter retry work; +- the existing duplicate-receipt guard remains unchanged. + +TDD evidence: the new pending-only regression failed before the source edit +(`Assert.Empty`, one receipt returned) and passes afterward. The production +controller/recenter regression and committed-invariant fail-fast regression +also pass. Focused `OriginRecenter` group: 20/20. + +Final gates: Release build passed; the complete Release suite passed +10,815/10,815 with 4 skips; the connected lifecycle gate passed at +`logs/connected-world-gate-20260802-203751/report.json`; and the nine-stop +soak passed at `logs/connected-r6-soak-20260802-204309.report.json` with all +9 canonical checkpoints, zero failures, zero wait cues, zero pending +landblock retirements, and zero recurrence of the 243x exception signature. + ### F3 addendum (coordinator resolution accepted, implemented) Hand-calls KEPT as honest documented models: enriched comments at all six item-6/8 sites (LiveEntityHydrationControllerTests x5 sites incl. the diff --git a/docs/research/2026-08-02-collision-throughput-handoff/p1-retirement-receipt-loop.md b/docs/research/2026-08-02-collision-throughput-handoff/p1-retirement-receipt-loop.md new file mode 100644 index 00000000..4621716a --- /dev/null +++ b/docs/research/2026-08-02-collision-throughput-handoff/p1-retirement-receipt-loop.md @@ -0,0 +1,95 @@ +# P1 — origin-recenter retirement-receipt loop + +## Observed failure + +`launch-feeltest-oclone.log` contains 243 consecutive failures with this +shape: + +```text +streaming: origin-recenter preparation will resume: +InvalidOperationException: Landblock 0xC85AFFFF already has a full +retirement receipt. +``` + +The stack is `StreamingController.TryAdvanceOriginRecenterPreparation` → +`LandblockPresentationPipeline.DetachAllForOriginRecenter` → +`LandblockRetirementCoordinator.AdoptDetachedFull`. + +## Root cause + +An ordinary full retirement detaches every landblock-owned presentation +resource first, then parks surviving live entities in +`GpuWorldState._pendingByLandblock` while the exact cleanup ticket advances +asynchronously (`GpuWorldState.DetachLandblock`, around lines 1188–1314). + +The origin-recenter swap incorrectly treated every pending-only live bucket +as another landblock presentation generation (`GpuWorldState.cs`, former +lines 1352–1353). It therefore emitted a second full cleanup receipt for the +same already-retired generation. `LandblockRetirementCoordinator` correctly +rejected that duplicate at lines 416–425. Because spatial detachment had +already committed, the broad retry catch in +`StreamingController.TryAdvanceOriginRecenterPreparation` then repeated the +detach against the changed state every frame. + +The pre-fix regression test +`OriginRecenterAdoption_PendingOnlyLiveProjectionDoesNotCreateSecondFullReceipt` +failed because the recenter returned one receipt for the pending-only bucket. + +## Retail and reference boundary + +Retail destroys one concrete landblock owner synchronously: +`CLandBlock::destroy_static_objects` (`0x0052FA50`) leaves and deletes the +landblock's static objects; `CLandBlock::Destroy` (`0x0052FAA0`) releases its +buildings and landblock data; `CLandBlock::release_all` (`0x0052FCF0`) +releases the landblock's object and visibility ownership. A live object +parked outside a loaded landblock is not a second `CLandBlock` and therefore +cannot create a second landblock-destruction transaction. + +The extracted WorldBuilder reference follows the same ownership boundary: +`ObjectRenderManagerBase` removes an actual `_landblocks` entry before +`UnloadLandblockResources`, and `PortalRenderManager` only unloads a removed +`PortalLandblock`. Neither treats an independently parked object as a new +landblock resource owner. + +Acdream retains its approved asynchronous adaptation: the first exact +receipt owns cleanup, while the live projection survives spatial recentering. + +## Fix + +- `GpuWorldState.DetachAllForOriginRecenter` no longer creates retirement + receipts from `_pendingByLandblock` alone. Pending live identities are + still captured from `_projectionLocations`, cleared atomically, and + re-parked unchanged. +- A landblock that also owns loaded, pending-render, pending-near, tier, or + bounds state still receives its exact full receipt. +- A receipt-ledger invariant thrown after spatial detachment is now surfaced + as a committed `StreamingMutationException`; it is terminal rather than + falsely logged as resumable work. +- The genuine duplicate-receipt guard remains unchanged. + +## Deterministic evidence + +- The new pending-only regression failed before the source fix and passes + afterward. +- `OriginRecenter_PendingOnlyLiveProjectionKeepsItsExistingRetirementOwner` + drives the production recenter/controller sequence and proves the origin + commits while the first cleanup ticket remains pending. +- `OriginRecenter_CommittedReceiptInvariantFailsFastInsteadOfReplayingDetach` + proves a genuine post-detach ledger violation surfaces once rather than + entering a frame-by-frame retry loop. +- The complete `OriginRecenter` focused group passes 20/20. + +## Gate evidence + +- Release build: 0 errors (21 pre-existing warnings). +- Complete Release suite: 10,815 passed, 0 failed, 4 skipped. +- Connected lifecycle/reconnect gate: + `logs/connected-world-gate-20260802-203751/report.json` — `Passed=true`. +- Connected nine-stop soak: + `logs/connected-r6-soak-20260802-204309.report.json` — `Passed=true`, + `Failures=[]`, graceful exit, all 9 canonical checkpoints present, no wait + cue, no pending landblock retirement, no reveal invariant failure, and no + render-shadow mismatch. +- The soak artifacts contain zero occurrences of + `already has a full retirement receipt`; the captured failing session had + 243. diff --git a/src/AcDream.App/Streaming/GpuWorldState.cs b/src/AcDream.App/Streaming/GpuWorldState.cs index 5b2a32cd..b19da86a 100644 --- a/src/AcDream.App/Streaming/GpuWorldState.cs +++ b/src/AcDream.App/Streaming/GpuWorldState.cs @@ -1349,8 +1349,14 @@ public sealed class GpuWorldState : ILiveEntitySpatialQuery if (id != 0u) AddId(id); } - foreach (uint id in _pendingByLandblock.Keys) - AddId(id); + // A pending-only bucket owns live spatial projections, not a loaded + // landblock presentation generation. Those projections are retained + // below through _projectionLocations, but there is no terrain, + // collision, static-script, or renderer owner to retire. In + // particular, DetachLandblock deliberately parks surviving live + // projections here while its existing exact retirement receipt is + // still advancing. Emitting another full receipt during a recenter + // would give the same retired generation two cleanup owners. foreach (uint id in _pendingRenderIdsByLandblock.Keys) AddId(id); foreach (uint id in _pendingNearTierLandblocks) diff --git a/src/AcDream.App/Streaming/LandblockPresentationPipeline.cs b/src/AcDream.App/Streaming/LandblockPresentationPipeline.cs index 75802436..93c019ec 100644 --- a/src/AcDream.App/Streaming/LandblockPresentationPipeline.cs +++ b/src/AcDream.App/Streaming/LandblockPresentationPipeline.cs @@ -322,8 +322,25 @@ public sealed class LandblockPresentationPipeline { GpuWorldRecenterRetirement detached = _state.DetachAllForOriginRecenter(); - Exception? adoptionFailure = - _retirements.AdoptDetachedFull(detached.Landblocks); + Exception? adoptionFailure; + try + { + adoptionFailure = + _retirements.AdoptDetachedFull(detached.Landblocks); + } + catch (Exception error) + { + // Spatial detachment is already committed. A receipt-ledger + // invariant failure cannot be retried by detaching the same + // generation again; doing so was the 243-frame origin-recenter + // exception loop. Surface the committed edge so the controller + // fails fast instead of pretending the operation is resumable. + throw new StreamingMutationException( + "Origin-recenter retirement receipt adoption failed after " + + "the spatial generation detached.", + mutationCommitted: true, + error); + } Exception? failure = (detached.ObserverFailure, adoptionFailure) switch { (null, null) => null, diff --git a/src/AcDream.App/Streaming/StreamingController.cs b/src/AcDream.App/Streaming/StreamingController.cs index b10e88e5..48a1ab81 100644 --- a/src/AcDream.App/Streaming/StreamingController.cs +++ b/src/AcDream.App/Streaming/StreamingController.cs @@ -1433,6 +1433,14 @@ public sealed class StreamingController transaction.PreparationCommitted = true; return true; } + catch (StreamingMutationException error) when (error.MutationCommitted) + { + // The old spatial generation is already gone. Re-entering this + // transaction would replay the detach against a new state, so a + // committed receipt/adoption invariant is terminal and must be + // surfaced to the caller. + throw; + } catch (Exception error) { Console.WriteLine( diff --git a/tests/AcDream.App.Tests/Streaming/GpuWorldStateVisibilityTests.cs b/tests/AcDream.App.Tests/Streaming/GpuWorldStateVisibilityTests.cs index fa87840f..e3d5bfd7 100644 --- a/tests/AcDream.App.Tests/Streaming/GpuWorldStateVisibilityTests.cs +++ b/tests/AcDream.App.Tests/Streaming/GpuWorldStateVisibilityTests.cs @@ -293,7 +293,7 @@ public sealed class GpuWorldStateVisibilityTests Assert.Null(result.ObserverFailure); Assert.Equal( - [firstLandblock, secondLandblock, pendingLandblock], + [firstLandblock, secondLandblock], result.Landblocks.Select(retirement => retirement.LandblockId)); Assert.Contains( result.Landblocks.Single( @@ -311,10 +311,9 @@ public sealed class GpuWorldStateVisibilityTests result.Landblocks.Single( retirement => retirement.LandblockId == secondLandblock).Entities, entity => ReferenceEquals(entity, player)); - Assert.Same( - pending, - Assert.Single(result.Landblocks.Single( - retirement => retirement.LandblockId == pendingLandblock).Entities)); + Assert.DoesNotContain( + result.Landblocks, + retirement => retirement.LandblockId == pendingLandblock); Assert.Empty(state.LoadedLandblockIds); Assert.Empty(state.Entities); diff --git a/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs b/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs index c7751b5f..fc9a3753 100644 --- a/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs +++ b/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs @@ -160,6 +160,112 @@ public sealed class LandblockPresentationPipelineTests Assert.Equal([0x2223FFFFu], enqueued); } + [Fact] + public void OriginRecenter_PendingOnlyLiveProjectionKeepsItsExistingRetirementOwner() + { + const uint oldLandblockId = 0x2424FFFFu; + var live = Entity(1u, serverGuid: 0x70000004u); + var state = new GpuWorldState(); + state.AddLandblock(new LoadedLandblock( + oldLandblockId, + new LandBlock(), + Array.Empty())); + state.PlaceLiveEntityProjection(oldLandblockId, live); + + bool holdCleanup = true; + var retirements = new LandblockRetirementCoordinator( + state, + ticket => ticket.RunOnce( + LandblockRetirementStage.EntityLighting, + () => + { + if (holdCleanup) + { + throw new InvalidOperationException( + "injected retained cleanup"); + } + }), + _ => LandblockRetirementStage.EntityLighting); + retirements.BeginFull(oldLandblockId); + Assert.Equal(1, retirements.PendingCount); + Assert.Equal(1, state.PendingLiveEntityCount); + + var origin = new LiveWorldOriginState(); + Assert.True(origin.TryInitialize(0x24, 0x24)); + var controller = new StreamingController( + enqueueLoad: static (_, _) => { }, + enqueueUnload: static _ => { }, + drainCompletions: static _ => Array.Empty(), + applyTerrain: static (_, _) => { }, + state, + nearRadius: 0, + farRadius: 0, + retirementCoordinator: retirements, + workBudgetOptions: GenerousWorkBudget()); + var recenter = new StreamingOriginRecenterCoordinator(controller, origin); + + Assert.False(recenter.Begin(0x25, 0x25, isSealedDungeon: false)); + Assert.True(Converge(recenter, controller, 0x24, 0x24)); + + Assert.Equal((0x25, 0x25), (origin.CenterX, origin.CenterY)); + Assert.Equal(1, retirements.PendingCount); + Assert.Equal(1, state.PendingLiveEntityCount); + + holdCleanup = false; + controller.Tick(0x25, 0x25); + Assert.Equal(0, retirements.PendingCount); + } + + [Fact] + public void OriginRecenter_CommittedReceiptInvariantFailsFastInsteadOfReplayingDetach() + { + const uint landblockId = 0x2626FFFFu; + var state = new GpuWorldState(); + state.AddLandblock(new LoadedLandblock( + landblockId, + new LandBlock(), + Array.Empty())); + var retirements = new LandblockRetirementCoordinator( + state, + ticket => ticket.RunOnce( + LandblockRetirementStage.EntityLighting, + static () => throw new InvalidOperationException( + "injected retained cleanup")), + _ => LandblockRetirementStage.EntityLighting); + retirements.BeginFull(landblockId); + Assert.Equal(1, retirements.PendingCount); + + // Directly violate the production publication fence so the recenter + // obtains a genuinely conflicting receipt after its spatial commit. + // The invariant must surface once; it must not become retry work. + state.AddLandblock(new LoadedLandblock( + landblockId, + new LandBlock(), + Array.Empty())); + + var origin = new LiveWorldOriginState(); + Assert.True(origin.TryInitialize(0x26, 0x26)); + var controller = new StreamingController( + enqueueLoad: static (_, _) => { }, + enqueueUnload: static _ => { }, + drainCompletions: static _ => Array.Empty(), + applyTerrain: static (_, _) => { }, + state, + nearRadius: 0, + farRadius: 0, + retirementCoordinator: retirements, + workBudgetOptions: GenerousWorkBudget()); + var recenter = new StreamingOriginRecenterCoordinator(controller, origin); + Assert.False(recenter.Begin(0x27, 0x27, isSealedDungeon: false)); + + StreamingMutationException error = Assert.Throws( + () => controller.Tick(0x26, 0x26)); + + Assert.True(error.MutationCommitted); + Assert.IsType(error.InnerException); + Assert.False(state.IsLoaded(landblockId)); + } + [Fact] public void OriginRecenter_DetachesFullTwentyFiveByTwentyFiveWindowAtomically() { diff --git a/tests/AcDream.App.Tests/Streaming/LandblockRetirementCoordinatorTests.cs b/tests/AcDream.App.Tests/Streaming/LandblockRetirementCoordinatorTests.cs index 3c8c84fe..0d006d85 100644 --- a/tests/AcDream.App.Tests/Streaming/LandblockRetirementCoordinatorTests.cs +++ b/tests/AcDream.App.Tests/Streaming/LandblockRetirementCoordinatorTests.cs @@ -595,6 +595,34 @@ public sealed class LandblockRetirementCoordinatorTests Assert.Equal(0, detachedCallbacks); } + [Fact] + public void OriginRecenterAdoption_PendingOnlyLiveProjectionDoesNotCreateSecondFullReceipt() + { + const uint landblockId = 0x4648FFFFu; + WorldEntity live = Entity(1, serverGuid: 0x70000003u); + GpuWorldState state = StateWith(landblockId, live); + LandblockRetirementCoordinator coordinator = + LandblockRetirementCoordinator.CreateBudgeted( + state, + ticket => AdvancePresentationStep(ticket), + ticket => CompletePresentation(ticket)); + + // The ordinary retirement has already detached every landblock-owned + // resource. The still-live entity is parked in the pending bucket + // while that exact cleanup receipt advances asynchronously. + coordinator.BeginFull(landblockId); + Assert.Equal(1, coordinator.PendingCount); + Assert.False(state.IsLoaded(landblockId)); + + GpuWorldRecenterRetirement recenter = + state.DetachAllForOriginRecenter(); + + Assert.Empty(recenter.Landblocks); + Assert.Null(coordinator.AdoptDetachedFull(recenter.Landblocks)); + Assert.Equal(1, coordinator.PendingCount); + Assert.True(coordinator.IsPending(landblockId)); + } + [Fact] public void OriginRecenterAdoption_ObserverFailureRetainsEveryCleanupReceipt() {