test(streaming): enforce slice E physical evidence

Record route-lifetime streaming overruns and maximum operation costs, require canonical reveal/resource convergence at every connected checkpoint, and keep recenter semantics independent of the production wall-clock budget.

Co-authored-by: Erik Nilsson <erikn@users.noreply.github.com>

src/AcDream.App/Streaming/StreamingController.cs

@@ -69,6 +69,12 @@ public sealed class StreamingController
     private long _nextCompletionSequence;
     private int _legacyCompletionProfile = 4;
     private DestinationReservation? _destinationReservation;
+    private long _lifetimeWorkOverruns;
+    private long _lifetimeOversizedProgress;
+    private double _maximumWorkFrameMilliseconds;
+    private string? _maximumWorkFrameStage;
+    private double _maximumWorkOperationMilliseconds;
+    private string? _maximumWorkOperationStage;
 
     private readonly GpuWorldState _state;
     private StreamingRegion? _region;
@@ -190,6 +196,12 @@ public sealed class StreamingController
                 _completionQueue.CaptureSnapshot();
             return new StreamingWorkDiagnostics(
                 _lastWorkMeter,
+                _lifetimeWorkOverruns,
+                _lifetimeOversizedProgress,
+                _maximumWorkFrameMilliseconds,
+                _maximumWorkFrameStage,
+                _maximumWorkOperationMilliseconds,
+                _maximumWorkOperationStage,
                 queued.Count,
                 queued.RetainedCpuBytes,
                 queued.OldestAgeMilliseconds,
@@ -690,11 +702,42 @@ public sealed class StreamingController
         finally
         {
             meter.FinishFrame();
-            _lastWorkMeter = meter.Snapshot;
+            StreamingWorkMeterSnapshot snapshot = meter.Snapshot;
+            ObserveWorkLifetime(snapshot);
+            _lastWorkMeter = snapshot;
             _activeWorkMeter = null;
         }
     }
 
+    private void ObserveWorkLifetime(StreamingWorkMeterSnapshot snapshot)
+    {
+        _lifetimeWorkOverruns = SaturatingAdd(
+            _lifetimeWorkOverruns,
+            snapshot.OverrunCount);
+        _lifetimeOversizedProgress = SaturatingAdd(
+            _lifetimeOversizedProgress,
+            snapshot.OversizedProgressCount);
+
+        if (_maximumWorkFrameStage is null
+            || snapshot.ElapsedMilliseconds > _maximumWorkFrameMilliseconds)
+        {
+            _maximumWorkFrameMilliseconds = snapshot.ElapsedMilliseconds;
+            _maximumWorkFrameStage = snapshot.LastStage;
+        }
+
+        if (_maximumWorkOperationStage is null
+            || snapshot.MaximumOperationMilliseconds
+                > _maximumWorkOperationMilliseconds)
+        {
+            _maximumWorkOperationMilliseconds =
+                snapshot.MaximumOperationMilliseconds;
+            _maximumWorkOperationStage = snapshot.MaximumOperationStage;
+        }
+    }
+
+    private static long SaturatingAdd(long left, int right) =>
+        left > long.MaxValue - right ? long.MaxValue : left + right;
+
     private void AdvanceRadiiReconfiguration()
     {
         if (_advancingRadiiReconfiguration)

src/AcDream.App/Streaming/StreamingWorkBudget.cs

@@ -155,6 +155,8 @@ public readonly record struct StreamingWorkMeterSnapshot(
     int OversizedProgressCount,
     int FailureCount,
     double ElapsedMilliseconds,
+    double MaximumOperationMilliseconds,
+    string? MaximumOperationStage,
     string? LastStage,
     StreamingWorkLimit LastLimit);
 
@@ -163,6 +165,12 @@ public readonly record struct StreamingWorkMeterSnapshot(
 /// </summary>
 public readonly record struct StreamingWorkDiagnostics(
     StreamingWorkMeterSnapshot LastFrame,
+    long LifetimeFrameOverrunCount,
+    long LifetimeOversizedProgressCount,
+    double MaximumFrameMilliseconds,
+    string? MaximumFrameStage,
+    double MaximumOperationMilliseconds,
+    string? MaximumOperationStage,
     int DeferredCompletions,
     long DeferredAdoptedCpuBytes,
     double OldestDeferredAgeMilliseconds,
@@ -193,6 +201,8 @@ public sealed class StreamingWorkMeter
     private long _destinationElapsedTicks;
     private long _nonDestinationElapsedTicks;
     private long _activeOperationStart;
+    private double _maximumOperationMilliseconds;
+    private string? _maximumOperationStage;
     private int _operations;
     private int _completed;
     private int _yields;
@@ -303,8 +313,8 @@ public sealed class StreamingWorkMeter
         CompleteLaneTiming();
         _completed++;
         _reservationActive = false;
-        _activeStage = null;
         ObserveElapsedOverrun();
+        _activeStage = null;
     }
 
     public void Fail()
@@ -313,8 +323,8 @@ public sealed class StreamingWorkMeter
         CompleteLaneTiming();
         _failures++;
         _reservationActive = false;
-        _activeStage = null;
         ObserveElapsedOverrun();
+        _activeStage = null;
     }
 
     public void FinishFrame()
@@ -341,6 +351,8 @@ public sealed class StreamingWorkMeter
                 _oversizedProgress,
                 _failures,
                 ElapsedMilliseconds(now),
+                _maximumOperationMilliseconds,
+                _maximumOperationStage,
                 _lastStage,
                 _lastLimit);
         }
@@ -428,6 +440,13 @@ public sealed class StreamingWorkMeter
     private void CompleteLaneTiming()
     {
         long raw = Math.Max(0L, _timestamp() - _activeOperationStart);
+        double elapsedMilliseconds = raw * 1000.0 / _frequency;
+        if (_maximumOperationStage is null
+            || elapsedMilliseconds > _maximumOperationMilliseconds)
+        {
+            _maximumOperationMilliseconds = elapsedMilliseconds;
+            _maximumOperationStage = _activeStage;
+        }
         long elapsed = raw > long.MaxValue / TimeSpan.TicksPerSecond
             ? long.MaxValue
             : raw * TimeSpan.TicksPerSecond / _frequency;

tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs

@@ -191,6 +191,46 @@ public sealed class ConnectedR6SoakContractTests
         foreach (string field in zeroFields)
             Assert.Contains($"'{field}'", source, StringComparison.Ordinal);
 
+        string[] revealFields =
+        [
+            "materialized",
+            "completed",
+            "worldViewportObserved",
+            "cancelled",
+            "invariantFailureCount",
+            "isReady",
+        ];
+        foreach (string field in revealFields)
+            Assert.Contains($"'{field}'", source, StringComparison.Ordinal);
+
+        string[] streamingZeroFields =
+        [
+            "deferredCompletions",
+            "deferredAdoptedCpuBytes",
+            "pendingPublications",
+            "pendingRetirements",
+            "workerCompletionBacklog",
+            "destinationBacklog",
+            "controlBacklog",
+            "unloadBacklog",
+            "nearBacklog",
+            "farBacklog",
+        ];
+        foreach (string field in streamingZeroFields)
+            Assert.Contains($"'{field}'", source, StringComparison.Ordinal);
+
+        string[] streamingEvidenceFields =
+        [
+            "lifetimeFrameOverrunCount",
+            "lifetimeOversizedProgressCount",
+            "maximumFrameMilliseconds",
+            "maximumFrameStage",
+            "maximumOperationMilliseconds",
+            "maximumOperationStage",
+        ];
+        foreach (string field in streamingEvidenceFields)
+            Assert.Contains($"'{field}'", source, StringComparison.Ordinal);
+
         Assert.Equal(1, CountOccurrences(
             source,
             "$privateLimit = [Math]::Max(192.0, $pair.First.PrivateMiB * 0.20)"));

tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs

@@ -633,7 +633,15 @@ public sealed class LandblockPresentationPipelineTests
             applyTerrain: (_, _) => publications++,
             state,
             nearRadius: 2,
-            farRadius: 4);
+            farRadius: 4,
+            workBudgetOptions: new StreamingWorkBudgetOptions(
+                MaxUpdateMilliseconds: 100,
+                MaxCompletionAdmissions: 64,
+                MaxAdoptedCpuBytes: 1_000_000,
+                MaxEntityOperations: 1_000,
+                MaxGpuUploadBytes: 1_000_000,
+                MaxGlRetireOperations: 1_000,
+                DestinationReserveFraction: 0.75f));
         var recenter = new StreamingOriginRecenterCoordinator(controller, origin);
 
         Assert.False(recenter.Begin(0x30, 0x30, isSealedDungeon: true));

tests/AcDream.App.Tests/Streaming/StreamingWorkBudgetTests.cs

@@ -241,6 +241,9 @@ public sealed class StreamingWorkBudgetTests
         Assert.Equal(1, snapshot.OverrunCount);
         Assert.Equal(1, snapshot.FailureCount);
         Assert.Equal(0, snapshot.CompletedOperations);
+        Assert.Equal(3, snapshot.MaximumOperationMilliseconds);
+        Assert.Equal("slow", snapshot.MaximumOperationStage);
+        Assert.Equal("slow", snapshot.LastStage);
         Assert.Equal(StreamingWorkLimit.Time, snapshot.LastLimit);
     }
 
@@ -465,6 +468,29 @@ public sealed class StreamingWorkBudgetTests
         Assert.Equal(4, second.LastFrame.CompletedOperations);
     }
 
+    [Fact]
+    public void ControllerPublishesLifetimeOversizeAndMaximumOperationEvidence()
+    {
+        uint center = StreamingRegion.EncodeLandblockIdForTest(32, 32);
+        var source = new QueueCompletionSource(Loaded(center));
+        StreamingController controller = Controller(
+            source,
+            static () => { },
+            WorkOptions(
+                admissions: 4,
+                cpuBytes: 1_000,
+                gpuBytes: 1));
+
+        controller.Tick(32, 32);
+
+        StreamingWorkDiagnostics diagnostics = controller.WorkDiagnostics;
+        Assert.True(diagnostics.LifetimeOversizedProgressCount >= 1);
+        Assert.True(diagnostics.MaximumOperationMilliseconds >= 0);
+        Assert.NotNull(diagnostics.MaximumOperationStage);
+        Assert.True(diagnostics.MaximumFrameMilliseconds >=
+            diagnostics.MaximumOperationMilliseconds);
+    }
+
     [Fact]
     public void ControllerBoundsProductionAdmissionByCountBeforeAdoption()
     {

tools/run-connected-r6-soak.ps1

@@ -336,6 +336,42 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
             $failures.Add("canonical checkpoint '$expectedName' came from process $($checkpoint.processId), expected $($Client.Id)")
         }
 
+        if ($null -eq $checkpoint.reveal) {
+            $failures.Add("canonical checkpoint '$expectedName' is missing reveal data")
+        }
+        else {
+            foreach ($field in @(
+                'materialized',
+                'completed',
+                'worldViewportObserved')) {
+                $property = $checkpoint.reveal.PSObject.Properties[$field]
+                if ($null -eq $property -or -not [bool]$property.Value) {
+                    $failures.Add("canonical checkpoint '$expectedName' has reveal.$field=false or missing")
+                }
+            }
+
+            $cancelled =
+                $checkpoint.reveal.PSObject.Properties['cancelled']
+            if ($null -eq $cancelled -or [bool]$cancelled.Value) {
+                $failures.Add("canonical checkpoint '$expectedName' has reveal.cancelled=true or missing")
+            }
+
+            $invariants =
+                $checkpoint.reveal.PSObject.Properties['invariantFailureCount']
+            if ($null -eq $invariants -or [long]$invariants.Value -ne 0) {
+                $failures.Add("canonical checkpoint '$expectedName' has reveal invariant failure(s)")
+            }
+
+            $readiness =
+                $checkpoint.reveal.PSObject.Properties['readiness']
+            $isReady = if ($null -ne $readiness) {
+                $readiness.Value.PSObject.Properties['isReady']
+            }
+            if ($null -eq $isReady -or -not [bool]$isReady.Value) {
+                $failures.Add("canonical checkpoint '$expectedName' has incomplete reveal readiness")
+            }
+        }
+
         if ($null -eq $checkpoint.render -or $null -eq $checkpoint.resources) {
             $failures.Add("canonical checkpoint '$expectedName' is missing render/resources data")
             continue
@@ -403,6 +439,45 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
                 $failures.Add("canonical checkpoint '$expectedName' has resources.$field=$($property.Value), expected zero")
             }
         }
+
+        $streamingWork =
+            $checkpoint.resources.PSObject.Properties['streamingWork']
+        if ($null -eq $streamingWork) {
+            $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork")
+        }
+        else {
+            foreach ($field in @(
+                'lifetimeFrameOverrunCount',
+                'lifetimeOversizedProgressCount',
+                'maximumFrameMilliseconds',
+                'maximumFrameStage',
+                'maximumOperationMilliseconds',
+                'maximumOperationStage')) {
+                if ($null -eq $streamingWork.Value.PSObject.Properties[$field]) {
+                    $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork.$field")
+                }
+            }
+
+            foreach ($field in @(
+                'deferredCompletions',
+                'deferredAdoptedCpuBytes',
+                'pendingPublications',
+                'pendingRetirements',
+                'workerCompletionBacklog',
+                'destinationBacklog',
+                'controlBacklog',
+                'unloadBacklog',
+                'nearBacklog',
+                'farBacklog')) {
+                $property = $streamingWork.Value.PSObject.Properties[$field]
+                if ($null -eq $property) {
+                    $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork.$field")
+                }
+                elseif ([long]$property.Value -ne 0) {
+                    $failures.Add("canonical checkpoint '$expectedName' has resources.streamingWork.$field=$($property.Value), expected zero")
+                }
+            }
+        }
     }
 
     if ($DenseTown) {