test(streaming): enforce slice E physical evidence

Record route-lifetime streaming overruns and maximum operation costs, require canonical reveal/resource convergence at every connected checkpoint, and keep recenter semantics independent of the production wall-clock budget. Co-authored-by: Erik Nilsson <erikn@users.noreply.github.com>
2026-07-24 19:46:17 +02:00 · 2026-07-24 19:46:17 +02:00 · 621f70eab6
commit 621f70eab6
parent 2ff8f844b0
6 changed files with 215 additions and 4 deletions
--- a/src/AcDream.App/Streaming/StreamingController.cs
+++ b/src/AcDream.App/Streaming/StreamingController.cs
@ -69,6 +69,12 @@ public sealed class StreamingController
    private long _nextCompletionSequence;
    private int _legacyCompletionProfile = 4;
    private DestinationReservation? _destinationReservation;
+    private long _lifetimeWorkOverruns;
+    private long _lifetimeOversizedProgress;
+    private double _maximumWorkFrameMilliseconds;
+    private string? _maximumWorkFrameStage;
+    private double _maximumWorkOperationMilliseconds;
+    private string? _maximumWorkOperationStage;

    private readonly GpuWorldState _state;
    private StreamingRegion? _region;
@ -190,6 +196,12 @@ public sealed class StreamingController
                _completionQueue.CaptureSnapshot();
            return new StreamingWorkDiagnostics(
                _lastWorkMeter,
+                _lifetimeWorkOverruns,
+                _lifetimeOversizedProgress,
+                _maximumWorkFrameMilliseconds,
+                _maximumWorkFrameStage,
+                _maximumWorkOperationMilliseconds,
+                _maximumWorkOperationStage,
                queued.Count,
                queued.RetainedCpuBytes,
                queued.OldestAgeMilliseconds,
@ -690,11 +702,42 @@ public sealed class StreamingController
        finally
        {
            meter.FinishFrame();
-            _lastWorkMeter = meter.Snapshot;
+            StreamingWorkMeterSnapshot snapshot = meter.Snapshot;
+            ObserveWorkLifetime(snapshot);
+            _lastWorkMeter = snapshot;
            _activeWorkMeter = null;
        }
    }

+    private void ObserveWorkLifetime(StreamingWorkMeterSnapshot snapshot)
+    {
+        _lifetimeWorkOverruns = SaturatingAdd(
+            _lifetimeWorkOverruns,
+            snapshot.OverrunCount);
+        _lifetimeOversizedProgress = SaturatingAdd(
+            _lifetimeOversizedProgress,
+            snapshot.OversizedProgressCount);
+
+        if (_maximumWorkFrameStage is null
+            || snapshot.ElapsedMilliseconds > _maximumWorkFrameMilliseconds)
+        {
+            _maximumWorkFrameMilliseconds = snapshot.ElapsedMilliseconds;
+            _maximumWorkFrameStage = snapshot.LastStage;
+        }
+
+        if (_maximumWorkOperationStage is null
+            || snapshot.MaximumOperationMilliseconds
+                > _maximumWorkOperationMilliseconds)
+        {
+            _maximumWorkOperationMilliseconds =
+                snapshot.MaximumOperationMilliseconds;
+            _maximumWorkOperationStage = snapshot.MaximumOperationStage;
+        }
+    }
+
+    private static long SaturatingAdd(long left, int right) =>
+        left > long.MaxValue - right ? long.MaxValue : left + right;
+
    private void AdvanceRadiiReconfiguration()
    {
        if (_advancingRadiiReconfiguration)
--- a/src/AcDream.App/Streaming/StreamingWorkBudget.cs
+++ b/src/AcDream.App/Streaming/StreamingWorkBudget.cs
@ -155,6 +155,8 @@ public readonly record struct StreamingWorkMeterSnapshot(
    int OversizedProgressCount,
    int FailureCount,
    double ElapsedMilliseconds,
+    double MaximumOperationMilliseconds,
+    string? MaximumOperationStage,
    string? LastStage,
    StreamingWorkLimit LastLimit);

@ -163,6 +165,12 @@ public readonly record struct StreamingWorkMeterSnapshot(
 /// </summary>
 public readonly record struct StreamingWorkDiagnostics(
    StreamingWorkMeterSnapshot LastFrame,
+    long LifetimeFrameOverrunCount,
+    long LifetimeOversizedProgressCount,
+    double MaximumFrameMilliseconds,
+    string? MaximumFrameStage,
+    double MaximumOperationMilliseconds,
+    string? MaximumOperationStage,
    int DeferredCompletions,
    long DeferredAdoptedCpuBytes,
    double OldestDeferredAgeMilliseconds,
@ -193,6 +201,8 @@ public sealed class StreamingWorkMeter
    private long _destinationElapsedTicks;
    private long _nonDestinationElapsedTicks;
    private long _activeOperationStart;
+    private double _maximumOperationMilliseconds;
+    private string? _maximumOperationStage;
    private int _operations;
    private int _completed;
    private int _yields;
@ -303,8 +313,8 @@ public sealed class StreamingWorkMeter
        CompleteLaneTiming();
        _completed++;
        _reservationActive = false;
-        _activeStage = null;
        ObserveElapsedOverrun();
+        _activeStage = null;
    }

    public void Fail()
@ -313,8 +323,8 @@ public sealed class StreamingWorkMeter
        CompleteLaneTiming();
        _failures++;
        _reservationActive = false;
-        _activeStage = null;
        ObserveElapsedOverrun();
+        _activeStage = null;
    }

    public void FinishFrame()
@ -341,6 +351,8 @@ public sealed class StreamingWorkMeter
                _oversizedProgress,
                _failures,
                ElapsedMilliseconds(now),
+                _maximumOperationMilliseconds,
+                _maximumOperationStage,
                _lastStage,
                _lastLimit);
        }
@ -428,6 +440,13 @@ public sealed class StreamingWorkMeter
    private void CompleteLaneTiming()
    {
        long raw = Math.Max(0L, _timestamp() - _activeOperationStart);
+        double elapsedMilliseconds = raw * 1000.0 / _frequency;
+        if (_maximumOperationStage is null
+            || elapsedMilliseconds > _maximumOperationMilliseconds)
+        {
+            _maximumOperationMilliseconds = elapsedMilliseconds;
+            _maximumOperationStage = _activeStage;
+        }
        long elapsed = raw > long.MaxValue / TimeSpan.TicksPerSecond
            ? long.MaxValue
            : raw * TimeSpan.TicksPerSecond / _frequency;
--- a/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs
+++ b/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs
@ -191,6 +191,46 @@ public sealed class ConnectedR6SoakContractTests
        foreach (string field in zeroFields)
            Assert.Contains($"'{field}'", source, StringComparison.Ordinal);

+        string[] revealFields =
+        [
+            "materialized",
+            "completed",
+            "worldViewportObserved",
+            "cancelled",
+            "invariantFailureCount",
+            "isReady",
+        ];
+        foreach (string field in revealFields)
+            Assert.Contains($"'{field}'", source, StringComparison.Ordinal);
+
+        string[] streamingZeroFields =
+        [
+            "deferredCompletions",
+            "deferredAdoptedCpuBytes",
+            "pendingPublications",
+            "pendingRetirements",
+            "workerCompletionBacklog",
+            "destinationBacklog",
+            "controlBacklog",
+            "unloadBacklog",
+            "nearBacklog",
+            "farBacklog",
+        ];
+        foreach (string field in streamingZeroFields)
+            Assert.Contains($"'{field}'", source, StringComparison.Ordinal);
+
+        string[] streamingEvidenceFields =
+        [
+            "lifetimeFrameOverrunCount",
+            "lifetimeOversizedProgressCount",
+            "maximumFrameMilliseconds",
+            "maximumFrameStage",
+            "maximumOperationMilliseconds",
+            "maximumOperationStage",
+        ];
+        foreach (string field in streamingEvidenceFields)
+            Assert.Contains($"'{field}'", source, StringComparison.Ordinal);
+
        Assert.Equal(1, CountOccurrences(
            source,
            "$privateLimit = [Math]::Max(192.0, $pair.First.PrivateMiB * 0.20)"));
--- a/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs
+++ b/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs
@ -633,7 +633,15 @@ public sealed class LandblockPresentationPipelineTests
            applyTerrain: (_, _) => publications++,
            state,
            nearRadius: 2,
-            farRadius: 4);
+            farRadius: 4,
+            workBudgetOptions: new StreamingWorkBudgetOptions(
+                MaxUpdateMilliseconds: 100,
+                MaxCompletionAdmissions: 64,
+                MaxAdoptedCpuBytes: 1_000_000,
+                MaxEntityOperations: 1_000,
+                MaxGpuUploadBytes: 1_000_000,
+                MaxGlRetireOperations: 1_000,
+                DestinationReserveFraction: 0.75f));
        var recenter = new StreamingOriginRecenterCoordinator(controller, origin);

        Assert.False(recenter.Begin(0x30, 0x30, isSealedDungeon: true));
--- a/tests/AcDream.App.Tests/Streaming/StreamingWorkBudgetTests.cs
+++ b/tests/AcDream.App.Tests/Streaming/StreamingWorkBudgetTests.cs
@ -241,6 +241,9 @@ public sealed class StreamingWorkBudgetTests
        Assert.Equal(1, snapshot.OverrunCount);
        Assert.Equal(1, snapshot.FailureCount);
        Assert.Equal(0, snapshot.CompletedOperations);
+        Assert.Equal(3, snapshot.MaximumOperationMilliseconds);
+        Assert.Equal("slow", snapshot.MaximumOperationStage);
+        Assert.Equal("slow", snapshot.LastStage);
        Assert.Equal(StreamingWorkLimit.Time, snapshot.LastLimit);
    }

@ -465,6 +468,29 @@ public sealed class StreamingWorkBudgetTests
        Assert.Equal(4, second.LastFrame.CompletedOperations);
    }

+    [Fact]
+    public void ControllerPublishesLifetimeOversizeAndMaximumOperationEvidence()
+    {
+        uint center = StreamingRegion.EncodeLandblockIdForTest(32, 32);
+        var source = new QueueCompletionSource(Loaded(center));
+        StreamingController controller = Controller(
+            source,
+            static () => { },
+            WorkOptions(
+                admissions: 4,
+                cpuBytes: 1_000,
+                gpuBytes: 1));
+
+        controller.Tick(32, 32);
+
+        StreamingWorkDiagnostics diagnostics = controller.WorkDiagnostics;
+        Assert.True(diagnostics.LifetimeOversizedProgressCount >= 1);
+        Assert.True(diagnostics.MaximumOperationMilliseconds >= 0);
+        Assert.NotNull(diagnostics.MaximumOperationStage);
+        Assert.True(diagnostics.MaximumFrameMilliseconds >=
+            diagnostics.MaximumOperationMilliseconds);
+    }
+
    [Fact]
    public void ControllerBoundsProductionAdmissionByCountBeforeAdoption()
    {
--- a/tools/run-connected-r6-soak.ps1
+++ b/tools/run-connected-r6-soak.ps1
@ -336,6 +336,42 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
            $failures.Add("canonical checkpoint '$expectedName' came from process $($checkpoint.processId), expected $($Client.Id)")
        }

+        if ($null -eq $checkpoint.reveal) {
+            $failures.Add("canonical checkpoint '$expectedName' is missing reveal data")
+        }
+        else {
+            foreach ($field in @(
+                'materialized',
+                'completed',
+                'worldViewportObserved')) {
+                $property = $checkpoint.reveal.PSObject.Properties[$field]
+                if ($null -eq $property -or -not [bool]$property.Value) {
+                    $failures.Add("canonical checkpoint '$expectedName' has reveal.$field=false or missing")
+                }
+            }
+
+            $cancelled =
+                $checkpoint.reveal.PSObject.Properties['cancelled']
+            if ($null -eq $cancelled -or [bool]$cancelled.Value) {
+                $failures.Add("canonical checkpoint '$expectedName' has reveal.cancelled=true or missing")
+            }
+
+            $invariants =
+                $checkpoint.reveal.PSObject.Properties['invariantFailureCount']
+            if ($null -eq $invariants -or [long]$invariants.Value -ne 0) {
+                $failures.Add("canonical checkpoint '$expectedName' has reveal invariant failure(s)")
+            }
+
+            $readiness =
+                $checkpoint.reveal.PSObject.Properties['readiness']
+            $isReady = if ($null -ne $readiness) {
+                $readiness.Value.PSObject.Properties['isReady']
+            }
+            if ($null -eq $isReady -or -not [bool]$isReady.Value) {
+                $failures.Add("canonical checkpoint '$expectedName' has incomplete reveal readiness")
+            }
+        }
+
        if ($null -eq $checkpoint.render -or $null -eq $checkpoint.resources) {
            $failures.Add("canonical checkpoint '$expectedName' is missing render/resources data")
            continue
@ -403,6 +439,45 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
                $failures.Add("canonical checkpoint '$expectedName' has resources.$field=$($property.Value), expected zero")
            }
        }
+
+        $streamingWork =
+            $checkpoint.resources.PSObject.Properties['streamingWork']
+        if ($null -eq $streamingWork) {
+            $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork")
+        }
+        else {
+            foreach ($field in @(
+                'lifetimeFrameOverrunCount',
+                'lifetimeOversizedProgressCount',
+                'maximumFrameMilliseconds',
+                'maximumFrameStage',
+                'maximumOperationMilliseconds',
+                'maximumOperationStage')) {
+                if ($null -eq $streamingWork.Value.PSObject.Properties[$field]) {
+                    $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork.$field")
+                }
+            }
+
+            foreach ($field in @(
+                'deferredCompletions',
+                'deferredAdoptedCpuBytes',
+                'pendingPublications',
+                'pendingRetirements',
+                'workerCompletionBacklog',
+                'destinationBacklog',
+                'controlBacklog',
+                'unloadBacklog',
+                'nearBacklog',
+                'farBacklog')) {
+                $property = $streamingWork.Value.PSObject.Properties[$field]
+                if ($null -eq $property) {
+                    $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork.$field")
+                }
+                elseif ([long]$property.Value -ne 0) {
+                    $failures.Add("canonical checkpoint '$expectedName' has resources.streamingWork.$field=$($property.Value), expected zero")
+                }
+            }
+        }
    }

    if ($DenseTown) {