From 621f70eab6d95bca44e4f53d469436288d25a56d Mon Sep 17 00:00:00 2001 From: Erik Date: Fri, 24 Jul 2026 19:46:17 +0200 Subject: [PATCH] test(streaming): enforce slice E physical evidence Record route-lifetime streaming overruns and maximum operation costs, require canonical reveal/resource convergence at every connected checkpoint, and keep recenter semantics independent of the production wall-clock budget. Co-authored-by: Erik Nilsson --- .../Streaming/StreamingController.cs | 45 ++++++++++- .../Streaming/StreamingWorkBudget.cs | 23 +++++- .../ConnectedR6SoakContractTests.cs | 40 ++++++++++ .../LandblockPresentationPipelineTests.cs | 10 ++- .../Streaming/StreamingWorkBudgetTests.cs | 26 +++++++ tools/run-connected-r6-soak.ps1 | 75 +++++++++++++++++++ 6 files changed, 215 insertions(+), 4 deletions(-) diff --git a/src/AcDream.App/Streaming/StreamingController.cs b/src/AcDream.App/Streaming/StreamingController.cs index 3f1c3e73..5b96cd1c 100644 --- a/src/AcDream.App/Streaming/StreamingController.cs +++ b/src/AcDream.App/Streaming/StreamingController.cs @@ -69,6 +69,12 @@ public sealed class StreamingController private long _nextCompletionSequence; private int _legacyCompletionProfile = 4; private DestinationReservation? _destinationReservation; + private long _lifetimeWorkOverruns; + private long _lifetimeOversizedProgress; + private double _maximumWorkFrameMilliseconds; + private string? _maximumWorkFrameStage; + private double _maximumWorkOperationMilliseconds; + private string? _maximumWorkOperationStage; private readonly GpuWorldState _state; private StreamingRegion? _region; @@ -190,6 +196,12 @@ public sealed class StreamingController _completionQueue.CaptureSnapshot(); return new StreamingWorkDiagnostics( _lastWorkMeter, + _lifetimeWorkOverruns, + _lifetimeOversizedProgress, + _maximumWorkFrameMilliseconds, + _maximumWorkFrameStage, + _maximumWorkOperationMilliseconds, + _maximumWorkOperationStage, queued.Count, queued.RetainedCpuBytes, queued.OldestAgeMilliseconds, @@ -690,11 +702,42 @@ public sealed class StreamingController finally { meter.FinishFrame(); - _lastWorkMeter = meter.Snapshot; + StreamingWorkMeterSnapshot snapshot = meter.Snapshot; + ObserveWorkLifetime(snapshot); + _lastWorkMeter = snapshot; _activeWorkMeter = null; } } + private void ObserveWorkLifetime(StreamingWorkMeterSnapshot snapshot) + { + _lifetimeWorkOverruns = SaturatingAdd( + _lifetimeWorkOverruns, + snapshot.OverrunCount); + _lifetimeOversizedProgress = SaturatingAdd( + _lifetimeOversizedProgress, + snapshot.OversizedProgressCount); + + if (_maximumWorkFrameStage is null + || snapshot.ElapsedMilliseconds > _maximumWorkFrameMilliseconds) + { + _maximumWorkFrameMilliseconds = snapshot.ElapsedMilliseconds; + _maximumWorkFrameStage = snapshot.LastStage; + } + + if (_maximumWorkOperationStage is null + || snapshot.MaximumOperationMilliseconds + > _maximumWorkOperationMilliseconds) + { + _maximumWorkOperationMilliseconds = + snapshot.MaximumOperationMilliseconds; + _maximumWorkOperationStage = snapshot.MaximumOperationStage; + } + } + + private static long SaturatingAdd(long left, int right) => + left > long.MaxValue - right ? long.MaxValue : left + right; + private void AdvanceRadiiReconfiguration() { if (_advancingRadiiReconfiguration) diff --git a/src/AcDream.App/Streaming/StreamingWorkBudget.cs b/src/AcDream.App/Streaming/StreamingWorkBudget.cs index 04d6c242..1d04f352 100644 --- a/src/AcDream.App/Streaming/StreamingWorkBudget.cs +++ b/src/AcDream.App/Streaming/StreamingWorkBudget.cs @@ -155,6 +155,8 @@ public readonly record struct StreamingWorkMeterSnapshot( int OversizedProgressCount, int FailureCount, double ElapsedMilliseconds, + double MaximumOperationMilliseconds, + string? MaximumOperationStage, string? LastStage, StreamingWorkLimit LastLimit); @@ -163,6 +165,12 @@ public readonly record struct StreamingWorkMeterSnapshot( /// public readonly record struct StreamingWorkDiagnostics( StreamingWorkMeterSnapshot LastFrame, + long LifetimeFrameOverrunCount, + long LifetimeOversizedProgressCount, + double MaximumFrameMilliseconds, + string? MaximumFrameStage, + double MaximumOperationMilliseconds, + string? MaximumOperationStage, int DeferredCompletions, long DeferredAdoptedCpuBytes, double OldestDeferredAgeMilliseconds, @@ -193,6 +201,8 @@ public sealed class StreamingWorkMeter private long _destinationElapsedTicks; private long _nonDestinationElapsedTicks; private long _activeOperationStart; + private double _maximumOperationMilliseconds; + private string? _maximumOperationStage; private int _operations; private int _completed; private int _yields; @@ -303,8 +313,8 @@ public sealed class StreamingWorkMeter CompleteLaneTiming(); _completed++; _reservationActive = false; - _activeStage = null; ObserveElapsedOverrun(); + _activeStage = null; } public void Fail() @@ -313,8 +323,8 @@ public sealed class StreamingWorkMeter CompleteLaneTiming(); _failures++; _reservationActive = false; - _activeStage = null; ObserveElapsedOverrun(); + _activeStage = null; } public void FinishFrame() @@ -341,6 +351,8 @@ public sealed class StreamingWorkMeter _oversizedProgress, _failures, ElapsedMilliseconds(now), + _maximumOperationMilliseconds, + _maximumOperationStage, _lastStage, _lastLimit); } @@ -428,6 +440,13 @@ public sealed class StreamingWorkMeter private void CompleteLaneTiming() { long raw = Math.Max(0L, _timestamp() - _activeOperationStart); + double elapsedMilliseconds = raw * 1000.0 / _frequency; + if (_maximumOperationStage is null + || elapsedMilliseconds > _maximumOperationMilliseconds) + { + _maximumOperationMilliseconds = elapsedMilliseconds; + _maximumOperationStage = _activeStage; + } long elapsed = raw > long.MaxValue / TimeSpan.TicksPerSecond ? long.MaxValue : raw * TimeSpan.TicksPerSecond / _frequency; diff --git a/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs b/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs index 60130c5e..e778775a 100644 --- a/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs +++ b/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs @@ -191,6 +191,46 @@ public sealed class ConnectedR6SoakContractTests foreach (string field in zeroFields) Assert.Contains($"'{field}'", source, StringComparison.Ordinal); + string[] revealFields = + [ + "materialized", + "completed", + "worldViewportObserved", + "cancelled", + "invariantFailureCount", + "isReady", + ]; + foreach (string field in revealFields) + Assert.Contains($"'{field}'", source, StringComparison.Ordinal); + + string[] streamingZeroFields = + [ + "deferredCompletions", + "deferredAdoptedCpuBytes", + "pendingPublications", + "pendingRetirements", + "workerCompletionBacklog", + "destinationBacklog", + "controlBacklog", + "unloadBacklog", + "nearBacklog", + "farBacklog", + ]; + foreach (string field in streamingZeroFields) + Assert.Contains($"'{field}'", source, StringComparison.Ordinal); + + string[] streamingEvidenceFields = + [ + "lifetimeFrameOverrunCount", + "lifetimeOversizedProgressCount", + "maximumFrameMilliseconds", + "maximumFrameStage", + "maximumOperationMilliseconds", + "maximumOperationStage", + ]; + foreach (string field in streamingEvidenceFields) + Assert.Contains($"'{field}'", source, StringComparison.Ordinal); + Assert.Equal(1, CountOccurrences( source, "$privateLimit = [Math]::Max(192.0, $pair.First.PrivateMiB * 0.20)")); diff --git a/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs b/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs index cbe595a3..60eace48 100644 --- a/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs +++ b/tests/AcDream.App.Tests/Streaming/LandblockPresentationPipelineTests.cs @@ -633,7 +633,15 @@ public sealed class LandblockPresentationPipelineTests applyTerrain: (_, _) => publications++, state, nearRadius: 2, - farRadius: 4); + farRadius: 4, + workBudgetOptions: new StreamingWorkBudgetOptions( + MaxUpdateMilliseconds: 100, + MaxCompletionAdmissions: 64, + MaxAdoptedCpuBytes: 1_000_000, + MaxEntityOperations: 1_000, + MaxGpuUploadBytes: 1_000_000, + MaxGlRetireOperations: 1_000, + DestinationReserveFraction: 0.75f)); var recenter = new StreamingOriginRecenterCoordinator(controller, origin); Assert.False(recenter.Begin(0x30, 0x30, isSealedDungeon: true)); diff --git a/tests/AcDream.App.Tests/Streaming/StreamingWorkBudgetTests.cs b/tests/AcDream.App.Tests/Streaming/StreamingWorkBudgetTests.cs index 4666719e..5241ea96 100644 --- a/tests/AcDream.App.Tests/Streaming/StreamingWorkBudgetTests.cs +++ b/tests/AcDream.App.Tests/Streaming/StreamingWorkBudgetTests.cs @@ -241,6 +241,9 @@ public sealed class StreamingWorkBudgetTests Assert.Equal(1, snapshot.OverrunCount); Assert.Equal(1, snapshot.FailureCount); Assert.Equal(0, snapshot.CompletedOperations); + Assert.Equal(3, snapshot.MaximumOperationMilliseconds); + Assert.Equal("slow", snapshot.MaximumOperationStage); + Assert.Equal("slow", snapshot.LastStage); Assert.Equal(StreamingWorkLimit.Time, snapshot.LastLimit); } @@ -465,6 +468,29 @@ public sealed class StreamingWorkBudgetTests Assert.Equal(4, second.LastFrame.CompletedOperations); } + [Fact] + public void ControllerPublishesLifetimeOversizeAndMaximumOperationEvidence() + { + uint center = StreamingRegion.EncodeLandblockIdForTest(32, 32); + var source = new QueueCompletionSource(Loaded(center)); + StreamingController controller = Controller( + source, + static () => { }, + WorkOptions( + admissions: 4, + cpuBytes: 1_000, + gpuBytes: 1)); + + controller.Tick(32, 32); + + StreamingWorkDiagnostics diagnostics = controller.WorkDiagnostics; + Assert.True(diagnostics.LifetimeOversizedProgressCount >= 1); + Assert.True(diagnostics.MaximumOperationMilliseconds >= 0); + Assert.NotNull(diagnostics.MaximumOperationStage); + Assert.True(diagnostics.MaximumFrameMilliseconds >= + diagnostics.MaximumOperationMilliseconds); + } + [Fact] public void ControllerBoundsProductionAdmissionByCountBeforeAdoption() { diff --git a/tools/run-connected-r6-soak.ps1 b/tools/run-connected-r6-soak.ps1 index 9111a48f..98aa595b 100644 --- a/tools/run-connected-r6-soak.ps1 +++ b/tools/run-connected-r6-soak.ps1 @@ -336,6 +336,42 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) { $failures.Add("canonical checkpoint '$expectedName' came from process $($checkpoint.processId), expected $($Client.Id)") } + if ($null -eq $checkpoint.reveal) { + $failures.Add("canonical checkpoint '$expectedName' is missing reveal data") + } + else { + foreach ($field in @( + 'materialized', + 'completed', + 'worldViewportObserved')) { + $property = $checkpoint.reveal.PSObject.Properties[$field] + if ($null -eq $property -or -not [bool]$property.Value) { + $failures.Add("canonical checkpoint '$expectedName' has reveal.$field=false or missing") + } + } + + $cancelled = + $checkpoint.reveal.PSObject.Properties['cancelled'] + if ($null -eq $cancelled -or [bool]$cancelled.Value) { + $failures.Add("canonical checkpoint '$expectedName' has reveal.cancelled=true or missing") + } + + $invariants = + $checkpoint.reveal.PSObject.Properties['invariantFailureCount'] + if ($null -eq $invariants -or [long]$invariants.Value -ne 0) { + $failures.Add("canonical checkpoint '$expectedName' has reveal invariant failure(s)") + } + + $readiness = + $checkpoint.reveal.PSObject.Properties['readiness'] + $isReady = if ($null -ne $readiness) { + $readiness.Value.PSObject.Properties['isReady'] + } + if ($null -eq $isReady -or -not [bool]$isReady.Value) { + $failures.Add("canonical checkpoint '$expectedName' has incomplete reveal readiness") + } + } + if ($null -eq $checkpoint.render -or $null -eq $checkpoint.resources) { $failures.Add("canonical checkpoint '$expectedName' is missing render/resources data") continue @@ -403,6 +439,45 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) { $failures.Add("canonical checkpoint '$expectedName' has resources.$field=$($property.Value), expected zero") } } + + $streamingWork = + $checkpoint.resources.PSObject.Properties['streamingWork'] + if ($null -eq $streamingWork) { + $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork") + } + else { + foreach ($field in @( + 'lifetimeFrameOverrunCount', + 'lifetimeOversizedProgressCount', + 'maximumFrameMilliseconds', + 'maximumFrameStage', + 'maximumOperationMilliseconds', + 'maximumOperationStage')) { + if ($null -eq $streamingWork.Value.PSObject.Properties[$field]) { + $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork.$field") + } + } + + foreach ($field in @( + 'deferredCompletions', + 'deferredAdoptedCpuBytes', + 'pendingPublications', + 'pendingRetirements', + 'workerCompletionBacklog', + 'destinationBacklog', + 'controlBacklog', + 'unloadBacklog', + 'nearBacklog', + 'farBacklog')) { + $property = $streamingWork.Value.PSObject.Properties[$field] + if ($null -eq $property) { + $failures.Add("canonical checkpoint '$expectedName' is missing resources.streamingWork.$field") + } + elseif ([long]$property.Value -ne 0) { + $failures.Add("canonical checkpoint '$expectedName' has resources.streamingWork.$field=$($property.Value), expected zero") + } + } + } } if ($DenseTown) {