From bca41487398347e54bd58aed264912fbca35bf9b Mon Sep 17 00:00:00 2001 From: Erik Date: Wed, 22 Jul 2026 20:01:06 +0200 Subject: [PATCH] test(app): add canonical connected soak snapshots Make scripted lifecycle checkpoints acknowledged post-diagnostics render barriers, capture the exact frame outcome beside canonical resource ownership, and harden the nine-stop route with ordered same-location cache and lifetime gates without weakening process residency thresholds. Co-authored-by: Codex --- ...dow-slice-8-checkpoint-k-canonical-soak.md | 17 +- .../Composition/FrameRootComposition.cs | 9 +- .../InteractionUiRuntimeSources.cs | 15 +- .../WorldLifecycleAutomationController.cs | 187 +++++++++++++-- .../WorldLifecycleResourceSnapshotSource.cs | 6 +- src/AcDream.App/Rendering/GameWindow.cs | 5 +- .../Rendering/RenderFrameOrchestrator.cs | 24 ++ .../Testing/RetailUiAutomationScriptRunner.cs | 73 +++++- .../Composition/FrameRootCompositionTests.cs | 8 + .../InteractionUiRuntimeSourcesTests.cs | 24 +- .../ConnectedR6SoakContractTests.cs | 133 +++++++++++ ...WorldLifecycleAutomationControllerTests.cs | 143 +++++++++++- .../Rendering/RenderFrameOrchestratorTests.cs | 48 +++- .../RenderFrameRecoveryIntegrationTests.cs | 1 + .../UI/RetailUiAutomationProbeTests.cs | 125 +++++++++- tools/connected-r6-soak.route.txt | 9 + tools/run-connected-r6-soak.ps1 | 220 ++++++++++++++++++ 17 files changed, 995 insertions(+), 52 deletions(-) create mode 100644 tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs diff --git a/docs/plans/2026-07-22-gamewindow-slice-8-checkpoint-k-canonical-soak.md b/docs/plans/2026-07-22-gamewindow-slice-8-checkpoint-k-canonical-soak.md index 59057af8..d7f2abdf 100644 --- a/docs/plans/2026-07-22-gamewindow-slice-8-checkpoint-k-canonical-soak.md +++ b/docs/plans/2026-07-22-gamewindow-slice-8-checkpoint-k-canonical-soak.md @@ -137,13 +137,16 @@ Every checkpoint hard-gates zero: - composite warmup work. The same-location Caul return → Caul plateau comparison requires exact equality -for deterministic owner/cache counts and bytes: loaded/visible/total landblocks, -materialized live owners, teardown/retirement queues, VFX binding/owner/script/ -light counts, mesh render-data/atlas/estimated bytes, staged work, tracked GPU -bytes/buffers/textures, and composite/particle texture ownership. Connected -`WorldEntities`, `AnimatedEntities`, `LiveEntities`, particles, and active -scripts may legitimately change with authoritative population or transient -effects; those deltas remain named workload warnings, not silent exclusions. +for deterministic cache/allocator state: loaded/visible/total landblocks, +teardown/retirement queues, mesh render-data/atlas/estimated bytes, staged work, +tracked GPU bytes/buffers/textures, and composite/particle texture ownership. +Connected `WorldEntities`, `AnimatedEntities`, `LiveEntities`, and materialized +live entities may legitimately change with authoritative population; those +deltas are named workload warnings. VFX binding/owner/script/light counts remain +hard equality gates when that population is stable, and become named workload +warnings when the authoritative population changed. Particles, emitters, and +active scripts are always transient workload warnings rather than silent +exclusions. Managed used/committed deltas are reported diagnostically. Existing working/private-memory, update-p95, and allocation-p50 thresholds remain byte- for-byte unchanged as the secondary residency/performance guard. diff --git a/src/AcDream.App/Composition/FrameRootComposition.cs b/src/AcDream.App/Composition/FrameRootComposition.cs index ff479f25..58851721 100644 --- a/src/AcDream.App/Composition/FrameRootComposition.cs +++ b/src/AcDream.App/Composition/FrameRootComposition.cs @@ -55,7 +55,6 @@ internal sealed record FrameRootDependencies( internal sealed record FrameRootResult( UpdateFrameOrchestrator Update, RenderFrameOrchestrator Render, - WorldLifecycleAutomationController? LifecycleAutomation, FrameRootRuntimeBindings RuntimeBindings, IDisposable FrameGraphPublication, AcDream.App.Net.LiveSessionHost SessionHost); @@ -420,7 +419,10 @@ internal sealed class FrameRootCompositionPhase artifactDirectory, message => d.Log("[UI-PROBE] " + message)); bindings.Adopt( - "world lifecycle automation", + "world lifecycle automation owner", + lifecycleAutomation); + bindings.Adopt( + "world lifecycle automation binding", interaction.LateBindings.Automation.Bind( lifecycleAutomation)); } @@ -453,6 +455,8 @@ internal sealed class FrameRootCompositionPhase worldSceneRenderer, privatePresentation, live.FrameDiagnostics, + (IRenderFramePostDiagnosticsPhase?)lifecycleAutomation + ?? NullRenderFramePostDiagnosticsPhase.Instance, (IRenderFrameFailureRecovery?)settings.DevTools?.Presenter ?? NullRenderFrameFailureRecovery.Instance); Fault(FrameRootCompositionPoint.RenderRootCreated); @@ -509,7 +513,6 @@ internal sealed class FrameRootCompositionPhase var result = new FrameRootResult( updateFrame, renderFrame, - lifecycleAutomation, bindings, frameGraphPublication, session.SessionHost); diff --git a/src/AcDream.App/Composition/InteractionUiRuntimeSources.cs b/src/AcDream.App/Composition/InteractionUiRuntimeSources.cs index 0c3d158b..3d303a5b 100644 --- a/src/AcDream.App/Composition/InteractionUiRuntimeSources.cs +++ b/src/AcDream.App/Composition/InteractionUiRuntimeSources.cs @@ -446,14 +446,25 @@ internal sealed class DeferredWorldLifecycleAutomationRuntime return new ExpectedOwnerBinding(target, Release); } - public bool TryWriteCheckpoint(string name, out string error) + public bool TryRequestCheckpoint( + string name, + out IRetailUiAutomationCheckpoint? checkpoint, + out string error) { if (!_deactivated && _target is { } target) - return target.TryWriteCheckpoint(name, out error); + return target.TryRequestCheckpoint(name, out checkpoint, out error); + checkpoint = null; error = "world lifecycle automation is not bound"; return false; } + public void CancelCheckpoint(IRetailUiAutomationCheckpoint checkpoint) + { + ArgumentNullException.ThrowIfNull(checkpoint); + if (!_deactivated && _target is { } target) + target.CancelCheckpoint(checkpoint); + } + public bool TryRequestScreenshot(string name, out string error) { if (!_deactivated && _target is { } target) diff --git a/src/AcDream.App/Diagnostics/WorldLifecycleAutomationController.cs b/src/AcDream.App/Diagnostics/WorldLifecycleAutomationController.cs index e6712a7f..33f5fe4b 100644 --- a/src/AcDream.App/Diagnostics/WorldLifecycleAutomationController.cs +++ b/src/AcDream.App/Diagnostics/WorldLifecycleAutomationController.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using AcDream.App.Rendering; using AcDream.App.Streaming; using AcDream.App.UI.Testing; @@ -71,14 +72,73 @@ internal sealed record WorldLifecycleCheckpoint( DateTime TimestampUtc, int ProcessId, WorldRevealLifecycleSnapshot Reveal, + RenderFrameOutcome Render, WorldLifecycleResourceSnapshot Resources); +internal sealed class WorldLifecycleCheckpointRequest : + IRetailUiAutomationCheckpoint +{ + private readonly object _sync = new(); + private RetailUiAutomationCheckpointStatus _status = + RetailUiAutomationCheckpointStatus.Pending; + private string? _error; + + public WorldLifecycleCheckpointRequest(object owner, int sequence, string name) + { + Owner = owner ?? throw new ArgumentNullException(nameof(owner)); + Sequence = sequence; + Name = name ?? throw new ArgumentNullException(nameof(name)); + } + + internal object Owner { get; } + public int Sequence { get; } + public string Name { get; } + + public RetailUiAutomationCheckpointStatus Status + { + get + { + lock (_sync) + return _status; + } + } + + public string? Error + { + get + { + lock (_sync) + return _error; + } + } + + internal bool TrySetTerminal( + RetailUiAutomationCheckpointStatus status, + string? error) + { + if (status == RetailUiAutomationCheckpointStatus.Pending) + throw new ArgumentOutOfRangeException(nameof(status)); + + lock (_sync) + { + if (_status != RetailUiAutomationCheckpointStatus.Pending) + return false; + _status = status; + _error = error; + return true; + } + } +} + /// /// Diagnostic-only runtime seam used by production retained-UI scripts. It /// writes one structured checkpoint at explicit script edges and delegates GL /// capture to the render-thread screenshot owner. /// -internal sealed class WorldLifecycleAutomationController : IRetailUiAutomationRuntime +internal sealed class WorldLifecycleAutomationController : + IRetailUiAutomationRuntime, + IRenderFramePostDiagnosticsPhase, + IDisposable { private static readonly JsonSerializerOptions JsonOptions = new() { @@ -87,16 +147,21 @@ internal sealed class WorldLifecycleAutomationController : IRetailUiAutomationRu private readonly Func _getReveal; private readonly Func _getPortalMaterializationCount; - private readonly Func _captureResources; + private readonly Func + _captureResources; private readonly FrameScreenshotController _screenshots; private readonly string _artifactDirectory; private readonly Action _log; + private readonly object _requestOwner = new(); + private readonly object _sync = new(); + private readonly Queue _requests = []; private int _sequence; + private bool _disposed; public WorldLifecycleAutomationController( Func getReveal, Func getPortalMaterializationCount, - Func captureResources, + Func captureResources, FrameScreenshotController screenshots, string artifactDirectory, Action? log = null) @@ -116,35 +181,127 @@ internal sealed class WorldLifecycleAutomationController : IRetailUiAutomationRu public bool IsWorldViewportVisible => _getReveal().WorldViewportObserved; public int PortalMaterializationCount => _getPortalMaterializationCount(); - public bool TryWriteCheckpoint(string name, out string error) + public bool TryRequestCheckpoint( + string name, + out IRetailUiAutomationCheckpoint? checkpoint, + out string error) { + checkpoint = null; if (!AutomationArtifactName.TryValidate(name, out error)) return false; + lock (_sync) + { + if (_disposed) + { + error = $"checkpoint '{name}' was rejected because world lifecycle automation is shutting down"; + return false; + } + + var request = new WorldLifecycleCheckpointRequest( + _requestOwner, + checked(++_sequence), + name); + _requests.Enqueue(request); + checkpoint = request; + error = string.Empty; + return true; + } + } + + public void CancelCheckpoint(IRetailUiAutomationCheckpoint checkpoint) + { + ArgumentNullException.ThrowIfNull(checkpoint); + if (checkpoint is not WorldLifecycleCheckpointRequest request + || !ReferenceEquals(request.Owner, _requestOwner)) + { + throw new ArgumentException( + "The checkpoint acknowledgement is not owned by this automation runtime.", + nameof(checkpoint)); + } + + lock (_sync) + { + request.TrySetTerminal( + RetailUiAutomationCheckpointStatus.Cancelled, + $"checkpoint '{request.Name}' was cancelled before render-frame capture"); + } + } + + public void Process(RenderFrameInput input, RenderFrameOutcome outcome) + { + lock (_sync) + { + if (_disposed) + return; + + while (_requests.Count != 0) + { + WorldLifecycleCheckpointRequest request = _requests.Dequeue(); + if (request.Status != RetailUiAutomationCheckpointStatus.Pending) + continue; + WriteCheckpoint(request, outcome); + } + } + } + + private void WriteCheckpoint( + WorldLifecycleCheckpointRequest request, + RenderFrameOutcome outcome) + { try { Directory.CreateDirectory(_artifactDirectory); var checkpoint = new WorldLifecycleCheckpoint( - Sequence: checked(++_sequence), - Name: name, + Sequence: request.Sequence, + Name: request.Name, TimestampUtc: DateTime.UtcNow, ProcessId: Environment.ProcessId, Reveal: _getReveal(), - Resources: _captureResources()); + Render: outcome, + Resources: _captureResources(outcome)); string json = JsonSerializer.Serialize(checkpoint, JsonOptions); - string timelinePath = Path.Combine(_artifactDirectory, "world-lifecycle.checkpoints.jsonl"); - File.AppendAllText(timelinePath, json + Environment.NewLine); + string timelinePath = Path.Combine( + _artifactDirectory, + "world-lifecycle.checkpoints.jsonl"); File.WriteAllText( - Path.Combine(_artifactDirectory, $"checkpoint-{name}.json"), + Path.Combine( + _artifactDirectory, + $"checkpoint-{request.Name}.json"), json + Environment.NewLine); - _log($"[world-gate] checkpoint name={name} sequence={checkpoint.Sequence} path={timelinePath}"); - error = string.Empty; - return true; + // Timeline publication is the commit edge: a visible row always + // has its complete named checkpoint artifact beside it. + File.AppendAllText(timelinePath, json + Environment.NewLine); + request.TrySetTerminal( + RetailUiAutomationCheckpointStatus.Succeeded, + error: null); + _log( + $"[world-gate] checkpoint name={request.Name} " + + $"sequence={request.Sequence} path={timelinePath}"); } catch (Exception exception) { - error = $"checkpoint '{name}' failed: {exception.Message}"; - return false; + request.TrySetTerminal( + RetailUiAutomationCheckpointStatus.Failed, + $"checkpoint '{request.Name}' sequence {request.Sequence} failed: " + + exception.Message); + } + } + + public void Dispose() + { + lock (_sync) + { + if (_disposed) + return; + _disposed = true; + while (_requests.Count != 0) + { + WorldLifecycleCheckpointRequest request = _requests.Dequeue(); + request.TrySetTerminal( + RetailUiAutomationCheckpointStatus.Cancelled, + $"checkpoint '{request.Name}' was cancelled during world lifecycle automation shutdown"); + } } } diff --git a/src/AcDream.App/Diagnostics/WorldLifecycleResourceSnapshotSource.cs b/src/AcDream.App/Diagnostics/WorldLifecycleResourceSnapshotSource.cs index 4c35aebd..c2d35bee 100644 --- a/src/AcDream.App/Diagnostics/WorldLifecycleResourceSnapshotSource.cs +++ b/src/AcDream.App/Diagnostics/WorldLifecycleResourceSnapshotSource.cs @@ -70,7 +70,7 @@ internal sealed class WorldLifecycleResourceSnapshotSource ?? throw new ArgumentNullException(nameof(frameProfiler)); } - public WorldLifecycleResourceSnapshot Capture() + public WorldLifecycleResourceSnapshot Capture(RenderFrameOutcome outcome) { ObjectMeshManager meshManager = _meshes.MeshManager ?? throw new InvalidOperationException( @@ -82,8 +82,8 @@ internal sealed class WorldLifecycleResourceSnapshotSource LoadedLandblocks: _world.LoadedLandblockIds.Count, WorldEntities: _world.Entities.Count, AnimatedEntities: _animations.Count, - VisibleLandblocks: render.VisibleLandblocks, - TotalLandblocks: render.TotalLandblocks, + VisibleLandblocks: outcome.World.VisibleLandblocks, + TotalLandblocks: outcome.World.TotalLandblocks, LiveEntities: _liveEntities.Count, MaterializedLiveEntities: _liveEntities.MaterializedCount, PendingLiveTeardowns: _liveEntities.PendingTeardownCount, diff --git a/src/AcDream.App/Rendering/GameWindow.cs b/src/AcDream.App/Rendering/GameWindow.cs index fe6574b1..a12abe6e 100644 --- a/src/AcDream.App/Rendering/GameWindow.cs +++ b/src/AcDream.App/Rendering/GameWindow.cs @@ -103,7 +103,6 @@ public sealed class GameWindow : private LivePresentationRuntimeBindings? _livePresentationBindings; private SessionPlayerRuntimeBindings? _sessionPlayerBindings; private AcDream.App.Diagnostics.FrameScreenshotController? _frameScreenshots; - private AcDream.App.Diagnostics.WorldLifecycleAutomationController? _worldLifecycleAutomation; private FrameRootRuntimeBindings? _frameRootBindings; private IDisposable? _frameGraphPublication; private AcDream.App.Rendering.GpuFrameFlightController? _gpuFrameFlights; @@ -1048,15 +1047,13 @@ public sealed class GameWindow : FrameRootResult result) { ArgumentNullException.ThrowIfNull(result); - if (_worldLifecycleAutomation is not null - || _frameRootBindings is not null + if (_frameRootBindings is not null || _frameGraphPublication is not null) { throw new InvalidOperationException( "The GameWindow composition shell already owns frame roots."); } - _worldLifecycleAutomation = result.LifecycleAutomation; _frameRootBindings = result.RuntimeBindings; _frameGraphPublication = result.FrameGraphPublication; } diff --git a/src/AcDream.App/Rendering/RenderFrameOrchestrator.cs b/src/AcDream.App/Rendering/RenderFrameOrchestrator.cs index 0d95c3a0..0ca3d915 100644 --- a/src/AcDream.App/Rendering/RenderFrameOrchestrator.cs +++ b/src/AcDream.App/Rendering/RenderFrameOrchestrator.cs @@ -60,6 +60,25 @@ internal interface IRenderFrameDiagnosticsPhase void Publish(RenderFrameInput input, RenderFrameOutcome outcome); } +internal interface IRenderFramePostDiagnosticsPhase +{ + void Process(RenderFrameInput input, RenderFrameOutcome outcome); +} + +internal sealed class NullRenderFramePostDiagnosticsPhase : + IRenderFramePostDiagnosticsPhase +{ + public static NullRenderFramePostDiagnosticsPhase Instance { get; } = new(); + + private NullRenderFramePostDiagnosticsPhase() + { + } + + public void Process(RenderFrameInput input, RenderFrameOutcome outcome) + { + } +} + /// /// Closes presentation transactions which may have opened before a later /// render phase failed. Recovery is idempotent so failures before the @@ -102,6 +121,7 @@ internal sealed class RenderFrameOrchestrator : IGameRenderFrameRoot private readonly IWorldSceneFramePhase _world; private readonly IPrivatePresentationFramePhase _presentation; private readonly IRenderFrameDiagnosticsPhase _diagnostics; + private readonly IRenderFramePostDiagnosticsPhase _postDiagnostics; private readonly IRenderFrameFailureRecovery _recovery; public RenderFrameOrchestrator( @@ -110,6 +130,7 @@ internal sealed class RenderFrameOrchestrator : IGameRenderFrameRoot IWorldSceneFramePhase world, IPrivatePresentationFramePhase presentation, IRenderFrameDiagnosticsPhase diagnostics, + IRenderFramePostDiagnosticsPhase postDiagnostics, IRenderFrameFailureRecovery recovery) { _lifetime = lifetime ?? throw new ArgumentNullException(nameof(lifetime)); @@ -117,6 +138,8 @@ internal sealed class RenderFrameOrchestrator : IGameRenderFrameRoot _world = world ?? throw new ArgumentNullException(nameof(world)); _presentation = presentation ?? throw new ArgumentNullException(nameof(presentation)); _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _postDiagnostics = postDiagnostics + ?? throw new ArgumentNullException(nameof(postDiagnostics)); _recovery = recovery ?? throw new ArgumentNullException(nameof(recovery)); } @@ -132,6 +155,7 @@ internal sealed class RenderFrameOrchestrator : IGameRenderFrameRoot _presentation.Render(input, world); var outcome = new RenderFrameOutcome(world, presentation); _diagnostics.Publish(input, outcome); + _postDiagnostics.Process(input, outcome); return outcome; } catch (Exception error) diff --git a/src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs b/src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs index 9262eb51..4fc0e1f7 100644 --- a/src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs +++ b/src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs @@ -7,6 +7,22 @@ using AcDream.UI.Abstractions.Input; namespace AcDream.App.UI.Testing; +public enum RetailUiAutomationCheckpointStatus +{ + Pending, + Succeeded, + Failed, + Cancelled, +} + +public interface IRetailUiAutomationCheckpoint +{ + int Sequence { get; } + string Name { get; } + RetailUiAutomationCheckpointStatus Status { get; } + string? Error { get; } +} + /// /// Narrow bridge from retained-UI scripts to render/world lifecycle /// diagnostics. Implementations run on the same update/render thread as the @@ -17,7 +33,11 @@ public interface IRetailUiAutomationRuntime bool IsWorldReady { get; } bool IsWorldViewportVisible { get; } int PortalMaterializationCount { get; } - bool TryWriteCheckpoint(string name, out string error); + bool TryRequestCheckpoint( + string name, + out IRetailUiAutomationCheckpoint? checkpoint, + out string error); + void CancelCheckpoint(IRetailUiAutomationCheckpoint checkpoint); bool TryRequestScreenshot(string name, out string error); bool IsScreenshotComplete(string name); } @@ -40,6 +60,8 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable private readonly HashSet _heldInputs = new(); private readonly bool _dumpOnStart; private readonly string? _loadError; + private IRetailUiAutomationCheckpoint? _checkpoint; + private int _checkpointCommandIndex = -1; private int _index; private int _activeIndex = -1; private double _commandElapsedMs; @@ -405,8 +427,47 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable return Stop(command, "usage: checkpoint "); if (_runtime is null) return Stop(command, "world lifecycle automation is unavailable"); - return _runtime.TryWriteCheckpoint(command.Parts[1], out string error) - || Stop(command, error); + + if (_checkpoint is null) + { + if (!_runtime.TryRequestCheckpoint( + command.Parts[1], + out IRetailUiAutomationCheckpoint? checkpoint, + out string error)) + { + return Stop(command, error); + } + + if (checkpoint is null) + { + return Stop( + command, + "world lifecycle automation accepted a checkpoint without returning its acknowledgement"); + } + + _checkpoint = checkpoint; + _checkpointCommandIndex = _index; + } + + if (_checkpointCommandIndex != _index) + { + return Stop( + command, + "checkpoint acknowledgement belongs to a different script command"); + } + + RetailUiAutomationCheckpointStatus status = _checkpoint.Status; + if (status == RetailUiAutomationCheckpointStatus.Pending) + return false; + + string? terminalError = _checkpoint.Error; + _checkpoint = null; + _checkpointCommandIndex = -1; + return status == RetailUiAutomationCheckpointStatus.Succeeded + || Stop( + command, + terminalError + ?? $"checkpoint '{command.Parts[1]}' ended with {status}"); } private bool DoScreenshot(ScriptCommand command) @@ -476,6 +537,12 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable public void Dispose() { if (_disposed) return; + if (_checkpoint is not null) + { + _runtime?.CancelCheckpoint(_checkpoint); + _checkpoint = null; + _checkpointCommandIndex = -1; + } ReleaseHeldInputs(); _disposed = true; } diff --git a/tests/AcDream.App.Tests/Composition/FrameRootCompositionTests.cs b/tests/AcDream.App.Tests/Composition/FrameRootCompositionTests.cs index b2af0485..99539f1e 100644 --- a/tests/AcDream.App.Tests/Composition/FrameRootCompositionTests.cs +++ b/tests/AcDream.App.Tests/Composition/FrameRootCompositionTests.cs @@ -39,7 +39,11 @@ public sealed class FrameRootCompositionTests source, "new RenderFrameResourceController(", "new WorldSceneRenderer(", + "new WorldLifecycleAutomationController(", + "\"world lifecycle automation owner\"", + "\"world lifecycle automation binding\"", "new RenderFrameOrchestrator(", + "(IRenderFramePostDiagnosticsPhase?)lifecycleAutomation", "new RetailLiveFrameCoordinator(", "new UpdateFrameOrchestrator(", "d.FrameGraphs.PublishOwned(", @@ -62,6 +66,7 @@ public sealed class FrameRootCompositionTests Assert.DoesNotContain("new AcDream.App.Rendering.WorldSceneRenderer(", source); Assert.DoesNotContain("new AcDream.App.Update.UpdateFrameOrchestrator(", source); Assert.DoesNotContain("CaptureWorldLifecycleResourceSnapshot", source); + Assert.DoesNotContain("_worldLifecycleAutomation", source); Assert.DoesNotContain("_frameGraphs.Publish(", source); } @@ -83,6 +88,9 @@ public sealed class FrameRootCompositionTests Assert.Contains("_liveEntities.PendingTeardownCount", snapshots); Assert.Contains("GpuMemoryTracker.AllocatedBytes", snapshots); Assert.Contains("_frameProfiler.LastReport", snapshots); + Assert.Contains("Capture(RenderFrameOutcome outcome)", snapshots); + Assert.Contains("outcome.World.VisibleLandblocks", snapshots); + Assert.Contains("outcome.World.TotalLandblocks", snapshots); } private sealed class RetryBinding( diff --git a/tests/AcDream.App.Tests/Composition/InteractionUiRuntimeSourcesTests.cs b/tests/AcDream.App.Tests/Composition/InteractionUiRuntimeSourcesTests.cs index 9bc42169..fdd57529 100644 --- a/tests/AcDream.App.Tests/Composition/InteractionUiRuntimeSourcesTests.cs +++ b/tests/AcDream.App.Tests/Composition/InteractionUiRuntimeSourcesTests.cs @@ -110,13 +110,14 @@ public sealed class InteractionUiRuntimeSourcesTests public void AutomationProxyIsInertUntilExactRuntimeBinds() { var source = new DeferredWorldLifecycleAutomationRuntime(); - Assert.False(source.TryWriteCheckpoint("early", out string earlyError)); + Assert.False(source.TryRequestCheckpoint( + "early", out _, out string earlyError)); Assert.Contains("not bound", earlyError); var target = new AutomationRuntime(); IDisposable binding = source.Bind(target); Assert.True(source.IsWorldReady); - Assert.True(source.TryWriteCheckpoint("ready", out _)); + Assert.True(source.TryRequestCheckpoint("ready", out _, out _)); Assert.Equal("ready", target.LastCheckpoint); binding.Dispose(); @@ -198,18 +199,35 @@ public sealed class InteractionUiRuntimeSourcesTests private sealed class AutomationRuntime : AcDream.App.UI.Testing.IRetailUiAutomationRuntime { + private sealed record CompletedCheckpoint(int Sequence, string Name) + : AcDream.App.UI.Testing.IRetailUiAutomationCheckpoint + { + public AcDream.App.UI.Testing.RetailUiAutomationCheckpointStatus Status => + AcDream.App.UI.Testing.RetailUiAutomationCheckpointStatus.Succeeded; + public string? Error => null; + } + public bool IsWorldReady => true; public bool IsWorldViewportVisible => true; public int PortalMaterializationCount => 2; public string? LastCheckpoint { get; private set; } - public bool TryWriteCheckpoint(string name, out string error) + public bool TryRequestCheckpoint( + string name, + out AcDream.App.UI.Testing.IRetailUiAutomationCheckpoint? checkpoint, + out string error) { LastCheckpoint = name; + checkpoint = new CompletedCheckpoint(1, name); error = string.Empty; return true; } + public void CancelCheckpoint( + AcDream.App.UI.Testing.IRetailUiAutomationCheckpoint checkpoint) + { + } + public bool TryRequestScreenshot(string name, out string error) { error = string.Empty; diff --git a/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs b/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs new file mode 100644 index 00000000..2b691633 --- /dev/null +++ b/tests/AcDream.App.Tests/Diagnostics/ConnectedR6SoakContractTests.cs @@ -0,0 +1,133 @@ +namespace AcDream.App.Tests.Diagnostics; + +public sealed class ConnectedR6SoakContractTests +{ + private static readonly string[] ExpectedCheckpointNames = + [ + "caul-baseline", + "sawato-baseline", + "rynthid", + "aerlinthe", + "sawato-return", + "holtburg", + "caul-return", + "sawato-plateau", + "caul-plateau", + ]; + + [Fact] + public void RouteContainsExactlyNineOrderedCheckpointBarriers() + { + string[] checkpoints = File.ReadAllLines(Path.Combine( + FindRepoRoot(), + "tools", + "connected-r6-soak.route.txt")) + .Select(static line => line.Trim()) + .Where(static line => line.StartsWith( + "checkpoint ", + StringComparison.Ordinal)) + .Select(static line => line["checkpoint ".Length..]) + .ToArray(); + + Assert.Equal(ExpectedCheckpointNames, checkpoints); + } + + [Fact] + public void GateConsumesCanonicalTimelineWithoutWeakeningResidencyFormulas() + { + string source = File.ReadAllText(Path.Combine( + FindRepoRoot(), + "tools", + "run-connected-r6-soak.ps1")); + + AssertAppearsInOrder( + source, + "$expectedCheckpointNames = @(", + "'caul-baseline'", + "'sawato-baseline'", + "'rynthid'", + "'aerlinthe'", + "'sawato-return'", + "'holtburg'", + "'caul-return'", + "'sawato-plateau'", + "'caul-plateau'"); + AssertAppearsInOrder( + source, + "$canonicalCheckpoints = @(Read-CanonicalCheckpoints)", + "Add-CanonicalCheckpointFailures $process", + "Add-RelativeGateFailures"); + Assert.Contains("CanonicalCheckpoints = @($canonicalCheckpoints)", source); + Assert.Contains("CheckpointTimeline = $checkpointTimeline", source); + Assert.Contains( + "Join-Path $artifactDir \"checkpoint-$expectedName.json\"", + source, + StringComparison.Ordinal); + Assert.Contains( + "missing render-world synchronization fields", + source, + StringComparison.Ordinal); + + string[] zeroFields = + [ + "pendingLiveTeardowns", + "pendingLandblockRetirements", + "stagedMeshUploads", + "stagedMeshBytes", + "compositeWarmupPending", + ]; + foreach (string field in zeroFields) + Assert.Contains($"'{field}'", source, StringComparison.Ordinal); + + Assert.Equal(1, CountOccurrences( + source, + "$privateLimit = [Math]::Max(192.0, $pair.First.PrivateMiB * 0.20)")); + Assert.Equal(1, CountOccurrences( + source, + "$workingLimit = [Math]::Max(192.0, $pair.First.WorkingSetMiB * 0.20)")); + Assert.Equal(1, CountOccurrences( + source, + "$updateLimit = $pair.First.UpdateP95Ms * 1.5 + 0.5")); + Assert.Equal(1, CountOccurrences( + source, + "$allocLimit = $pair.First.AllocP50Kb * 1.5 + 16.0")); + } + + private static int CountOccurrences(string source, string value) + { + int count = 0; + int cursor = 0; + while ((cursor = source.IndexOf(value, cursor, StringComparison.Ordinal)) >= 0) + { + count++; + cursor += value.Length; + } + + return count; + } + + private static void AssertAppearsInOrder(string source, params string[] values) + { + int cursor = -1; + foreach (string value in values) + { + int next = source.IndexOf(value, cursor + 1, StringComparison.Ordinal); + Assert.True(next >= 0, $"Missing expected source fragment: {value}"); + Assert.True(next > cursor, $"Out-of-order source fragment: {value}"); + cursor = next; + } + } + + private static string FindRepoRoot() + { + DirectoryInfo? directory = new(AppContext.BaseDirectory); + while (directory is not null) + { + if (File.Exists(Path.Combine(directory.FullName, "AcDream.slnx"))) + return directory.FullName; + directory = directory.Parent; + } + + throw new DirectoryNotFoundException("Could not find AcDream.slnx."); + } +} diff --git a/tests/AcDream.App.Tests/Diagnostics/WorldLifecycleAutomationControllerTests.cs b/tests/AcDream.App.Tests/Diagnostics/WorldLifecycleAutomationControllerTests.cs index 28000eda..1dad6542 100644 --- a/tests/AcDream.App.Tests/Diagnostics/WorldLifecycleAutomationControllerTests.cs +++ b/tests/AcDream.App.Tests/Diagnostics/WorldLifecycleAutomationControllerTests.cs @@ -1,12 +1,21 @@ using System.Text.Json; using AcDream.App.Diagnostics; +using AcDream.App.Rendering; using AcDream.App.Streaming; +using AcDream.App.UI.Testing; using SixLabors.ImageSharp; namespace AcDream.App.Tests.Diagnostics; public sealed class WorldLifecycleAutomationControllerTests { + private static readonly RenderFrameInput FrameInput = new(1d / 60d, 1280, 720); + private static readonly RenderFrameOutcome FrameOutcome = new( + new WorldRenderFrameOutcome(4, 17, NormalWorldDrawn: true), + new PrivatePresentationFrameOutcome( + PortalViewportDrawn: false, + ScreenshotCaptured: true)); + [Fact] public void ScreenshotCapture_FlipsGlRowsAndWritesACompletePng() { @@ -112,7 +121,7 @@ public sealed class WorldLifecycleAutomationControllerTests var controller = new WorldLifecycleAutomationController( () => reveal, () => 3, - () => resources, + _ => resources, screenshots, directory); @@ -121,13 +130,26 @@ public sealed class WorldLifecycleAutomationControllerTests Assert.True(controller.IsWorldReady); Assert.True(controller.IsWorldViewportVisible); Assert.Equal(3, controller.PortalMaterializationCount); - Assert.True(controller.TryWriteCheckpoint("dungeon", out string error), error); + Assert.True(controller.TryRequestCheckpoint( + "dungeon", + out IRetailUiAutomationCheckpoint? request, + out string error), error); + Assert.NotNull(request); + Assert.Equal(RetailUiAutomationCheckpointStatus.Pending, request.Status); + + controller.Process(FrameInput, FrameOutcome); + + Assert.Equal(RetailUiAutomationCheckpointStatus.Succeeded, request.Status); string line = Assert.Single(File.ReadAllLines( Path.Combine(directory, "world-lifecycle.checkpoints.jsonl"))); using JsonDocument json = JsonDocument.Parse(line); Assert.Equal("dungeon", json.RootElement.GetProperty("name").GetString()); Assert.Equal(7, json.RootElement.GetProperty("reveal").GetProperty("generation").GetInt64()); + Assert.Equal(4, json.RootElement.GetProperty("render").GetProperty("world") + .GetProperty("visibleLandblocks").GetInt32()); + Assert.True(json.RootElement.GetProperty("render").GetProperty("presentation") + .GetProperty("screenshotCaptured").GetBoolean()); Assert.Equal(42, json.RootElement.GetProperty("resources").GetProperty("worldEntities").GetInt32()); Assert.True(File.Exists(Path.Combine(directory, "checkpoint-dungeon.json"))); } @@ -137,6 +159,123 @@ public sealed class WorldLifecycleAutomationControllerTests } } + [Fact] + public void CheckpointRequests_DrainInFifoOrderOnce() + { + string directory = NewDirectory(); + int captures = 0; + var controller = CreateController( + directory, + _ => + { + captures++; + return EmptyResources(); + }); + + try + { + Assert.True(controller.TryRequestCheckpoint( + "first", out IRetailUiAutomationCheckpoint? first, out _)); + Assert.True(controller.TryRequestCheckpoint( + "second", out IRetailUiAutomationCheckpoint? second, out _)); + + controller.Process(FrameInput, FrameOutcome); + controller.Process(FrameInput, FrameOutcome); + + Assert.NotNull(first); + Assert.NotNull(second); + Assert.Equal(1, first.Sequence); + Assert.Equal(2, second.Sequence); + Assert.Equal(RetailUiAutomationCheckpointStatus.Succeeded, first.Status); + Assert.Equal(RetailUiAutomationCheckpointStatus.Succeeded, second.Status); + Assert.Equal(2, captures); + string[] lines = File.ReadAllLines( + Path.Combine(directory, "world-lifecycle.checkpoints.jsonl")); + Assert.Equal(2, lines.Length); + Assert.Contains("\"name\":\"first\"", lines[0], StringComparison.Ordinal); + Assert.Contains("\"name\":\"second\"", lines[1], StringComparison.Ordinal); + } + finally + { + controller.Dispose(); + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void CheckpointWriteFailure_IsReportedOnTokenWithoutThrowingFromFrame() + { + string directory = NewDirectory(); + string blockedPath = Path.Combine(directory, "not-a-directory"); + File.WriteAllText(blockedPath, "occupied"); + var controller = CreateController(blockedPath, _ => EmptyResources()); + + try + { + Assert.True(controller.TryRequestCheckpoint( + "failed", out IRetailUiAutomationCheckpoint? request, out _)); + + controller.Process(FrameInput, FrameOutcome); + + Assert.NotNull(request); + Assert.Equal(RetailUiAutomationCheckpointStatus.Failed, request.Status); + Assert.Contains("checkpoint 'failed' sequence 1 failed", request.Error); + } + finally + { + controller.Dispose(); + Directory.Delete(directory, recursive: true); + } + } + + [Fact] + public void CheckpointCancellationAndShutdown_AreExplicitAndNeverWritten() + { + string directory = NewDirectory(); + var controller = CreateController(directory, _ => EmptyResources()); + + try + { + Assert.True(controller.TryRequestCheckpoint( + "caller", out IRetailUiAutomationCheckpoint? caller, out _)); + Assert.True(controller.TryRequestCheckpoint( + "shutdown", out IRetailUiAutomationCheckpoint? shutdown, out _)); + Assert.NotNull(caller); + Assert.NotNull(shutdown); + + controller.CancelCheckpoint(caller); + controller.Dispose(); + controller.Process(FrameInput, FrameOutcome); + + Assert.Equal(RetailUiAutomationCheckpointStatus.Cancelled, caller.Status); + Assert.Contains("cancelled before render-frame capture", caller.Error); + Assert.Equal(RetailUiAutomationCheckpointStatus.Cancelled, shutdown.Status); + Assert.Contains("cancelled during world lifecycle automation shutdown", shutdown.Error); + Assert.False(controller.TryRequestCheckpoint( + "late", out IRetailUiAutomationCheckpoint? late, out string error)); + Assert.Null(late); + Assert.Contains("shutting down", error); + Assert.False(File.Exists(Path.Combine( + directory, + "world-lifecycle.checkpoints.jsonl"))); + } + finally + { + controller.Dispose(); + Directory.Delete(directory, recursive: true); + } + } + + private static WorldLifecycleAutomationController CreateController( + string directory, + Func capture) => + new( + () => default, + () => 0, + capture, + new FrameScreenshotController((_, _) => [], directory), + directory); + private static WorldLifecycleResourceSnapshot EmptyResources() => new( 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0L, 0, 0L, 0L, 0, 0, 0, 0, 0, 0, 0, 0L, 0L, 0d, 0d, null); diff --git a/tests/AcDream.App.Tests/Rendering/RenderFrameOrchestratorTests.cs b/tests/AcDream.App.Tests/Rendering/RenderFrameOrchestratorTests.cs index 34807786..43539e70 100644 --- a/tests/AcDream.App.Tests/Rendering/RenderFrameOrchestratorTests.cs +++ b/tests/AcDream.App.Tests/Rendering/RenderFrameOrchestratorTests.cs @@ -38,7 +38,10 @@ public sealed class RenderFrameOrchestratorTests RenderFrameOutcome outcome = orchestrator.Render(Input); Assert.Equal( - ["gpu-begin", "resources", "world", "presentation", "diagnostics", "gpu-end"], + [ + "gpu-begin", "resources", "world", "presentation", + "diagnostics", "post-diagnostics", "gpu-end", + ], calls); Assert.Equal(phases.World, outcome.World); Assert.Equal(phases.Presentation, outcome.Presentation); @@ -48,6 +51,7 @@ public sealed class RenderFrameOrchestratorTests ("world", Input), ("presentation", Input), ("diagnostics", Input), + ("post-diagnostics", Input), ], phases.ObservedInputs); Assert.Equal(phases.World, phases.ObservedWorld); @@ -77,6 +81,7 @@ public sealed class RenderFrameOrchestratorTests [InlineData("world")] [InlineData("presentation")] [InlineData("diagnostics")] + [InlineData("post-diagnostics")] public void RenderFailure_ClosesExactlyOnceAndPropagates(string failurePoint) { var calls = new List(); @@ -112,7 +117,10 @@ public sealed class RenderFrameOrchestratorTests Assert.Same(expected, actual); Assert.Equal( - ["gpu-begin", "resources", "world", "presentation", "diagnostics", "gpu-end"], + [ + "gpu-begin", "resources", "world", "presentation", + "diagnostics", "post-diagnostics", "gpu-end", + ], calls); } @@ -121,6 +129,7 @@ public sealed class RenderFrameOrchestratorTests [InlineData("world")] [InlineData("presentation")] [InlineData("diagnostics")] + [InlineData("post-diagnostics")] public void RenderAndCloseFailure_AreAggregatedInCausalOrder(string failurePoint) { var calls = new List(); @@ -208,17 +217,19 @@ public sealed class RenderFrameOrchestratorTests var phases = new RecordingPhases([]); Assert.Throws(() => new RenderFrameOrchestrator( - null!, phases, phases, phases, phases, phases)); + null!, phases, phases, phases, phases, phases, phases)); Assert.Throws(() => new RenderFrameOrchestrator( - phases, null!, phases, phases, phases, phases)); + phases, null!, phases, phases, phases, phases, phases)); Assert.Throws(() => new RenderFrameOrchestrator( - phases, phases, null!, phases, phases, phases)); + phases, phases, null!, phases, phases, phases, phases)); Assert.Throws(() => new RenderFrameOrchestrator( - phases, phases, phases, null!, phases, phases)); + phases, phases, phases, null!, phases, phases, phases)); Assert.Throws(() => new RenderFrameOrchestrator( - phases, phases, phases, phases, null!, phases)); + phases, phases, phases, phases, null!, phases, phases)); Assert.Throws(() => new RenderFrameOrchestrator( - phases, phases, phases, phases, phases, null!)); + phases, phases, phases, phases, phases, null!, phases)); + Assert.Throws(() => new RenderFrameOrchestrator( + phases, phases, phases, phases, phases, phases, null!)); } [Fact] @@ -231,6 +242,7 @@ public sealed class RenderFrameOrchestratorTests typeof(IWorldSceneFramePhase), typeof(IPrivatePresentationFramePhase), typeof(IRenderFrameDiagnosticsPhase), + typeof(IRenderFramePostDiagnosticsPhase), typeof(IRenderFrameFailureRecovery), ]; FieldInfo[] fields = typeof(RenderFrameOrchestrator).GetFields( @@ -308,6 +320,11 @@ public sealed class RenderFrameOrchestratorTests "gpu-begin", "resources", "world", "presentation", "diagnostics", "recovery-abort", "gpu-end", ], + "post-diagnostics" => + [ + "gpu-begin", "resources", "world", "presentation", "diagnostics", + "post-diagnostics", "recovery-abort", "gpu-end", + ], _ => throw new ArgumentOutOfRangeException(nameof(failurePoint)), }; @@ -320,13 +337,18 @@ public sealed class RenderFrameOrchestratorTests "world" => ["resources", "world"], "presentation" => ["resources", "world", "presentation"], "diagnostics" => ["resources", "world", "presentation", "diagnostics"], + "post-diagnostics" => + [ + "resources", "world", "presentation", "diagnostics", + "post-diagnostics", + ], _ => throw new ArgumentOutOfRangeException(nameof(failurePoint)), }; return phases.Select(phase => (phase, Input)).ToArray(); } private static RenderFrameOrchestrator Create(RecordingPhases phases) => - new(phases, phases, phases, phases, phases, phases); + new(phases, phases, phases, phases, phases, phases, phases); private sealed class RecordingPhases : IRenderFrameLifetime, @@ -334,6 +356,7 @@ public sealed class RenderFrameOrchestratorTests IWorldSceneFramePhase, IPrivatePresentationFramePhase, IRenderFrameDiagnosticsPhase, + IRenderFramePostDiagnosticsPhase, IRenderFrameFailureRecovery { private readonly List _calls; @@ -399,6 +422,13 @@ public sealed class RenderFrameOrchestratorTests Record("diagnostics"); } + public void Process(RenderFrameInput input, RenderFrameOutcome outcome) + { + ObservedInputs.Add(("post-diagnostics", input)); + ObservedOutcome = outcome; + Record("post-diagnostics"); + } + private void Record(string call) { _calls.Add(call); diff --git a/tests/AcDream.App.Tests/Rendering/RenderFrameRecoveryIntegrationTests.cs b/tests/AcDream.App.Tests/Rendering/RenderFrameRecoveryIntegrationTests.cs index ec0a10a5..9d27e27d 100644 --- a/tests/AcDream.App.Tests/Rendering/RenderFrameRecoveryIntegrationTests.cs +++ b/tests/AcDream.App.Tests/Rendering/RenderFrameRecoveryIntegrationTests.cs @@ -37,6 +37,7 @@ public sealed class RenderFrameRecoveryIntegrationTests new World(failure), presentation, new Diagnostics(), + NullRenderFramePostDiagnosticsPhase.Instance, devTools); Assert.Throws(() => orchestrator.Render(Input)); diff --git a/tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs b/tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs index 337f0477..e1b0bc00 100644 --- a/tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs +++ b/tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs @@ -26,6 +26,18 @@ public sealed class RetailUiAutomationProbeTests private sealed class FakeRuntime : IRetailUiAutomationRuntime { + private sealed class FakeCheckpoint(int sequence, string name) : + IRetailUiAutomationCheckpoint + { + public int Sequence { get; } = sequence; + public string Name { get; } = name; + public RetailUiAutomationCheckpointStatus Status { get; set; } = + RetailUiAutomationCheckpointStatus.Pending; + public string? Error { get; set; } + } + + private readonly List _checkpointTokens = []; + public bool IsWorldReady { get; set; } public bool IsWorldViewportVisible { get; set; } public int PortalMaterializationCount { get; set; } @@ -33,13 +45,41 @@ public sealed class RetailUiAutomationProbeTests public HashSet ScreenshotRequests { get; } = new(); public HashSet CompletedScreenshots { get; } = new(); - public bool TryWriteCheckpoint(string name, out string error) + public bool TryRequestCheckpoint( + string name, + out IRetailUiAutomationCheckpoint? checkpoint, + out string error) { Checkpoints.Add(name); + var token = new FakeCheckpoint(_checkpointTokens.Count + 1, name); + _checkpointTokens.Add(token); + checkpoint = token; error = string.Empty; return true; } + public void CancelCheckpoint(IRetailUiAutomationCheckpoint checkpoint) + { + var token = Assert.IsType(checkpoint); + token.Status = RetailUiAutomationCheckpointStatus.Cancelled; + token.Error = $"checkpoint '{token.Name}' cancelled"; + } + + public void CompleteCheckpoint( + string name, + RetailUiAutomationCheckpointStatus status = + RetailUiAutomationCheckpointStatus.Succeeded, + string? error = null) + { + FakeCheckpoint token = Assert.Single( + _checkpointTokens, + value => value.Name == name); + if (token.Status != RetailUiAutomationCheckpointStatus.Pending) + throw new InvalidOperationException("checkpoint is already terminal"); + token.Status = status; + token.Error = error; + } + public bool TryRequestScreenshot(string name, out string error) { ScreenshotRequests.Add(name); @@ -590,6 +630,16 @@ public sealed class RetailUiAutomationProbeTests runner.Tick(0.001d); Assert.Equal(["stable"], runtime.Checkpoints); + Assert.DoesNotContain("stable", runtime.ScreenshotRequests); + Assert.False(runner.Completed); + + runner.Tick(0.001d); + Assert.Equal(["stable"], runtime.Checkpoints); + Assert.DoesNotContain("stable", runtime.ScreenshotRequests); + + runtime.CompleteCheckpoint("stable"); + runner.Tick(0.001d); + Assert.Contains("stable", runtime.ScreenshotRequests); Assert.False(runner.Completed); @@ -604,6 +654,79 @@ public sealed class RetailUiAutomationProbeTests } } + [Theory] + [InlineData(RetailUiAutomationCheckpointStatus.Failed, "deferred write failed")] + [InlineData(RetailUiAutomationCheckpointStatus.Cancelled, "shutdown cancelled")] + public void ScriptRunner_checkpointTerminalFailure_StopsWithExactError( + RetailUiAutomationCheckpointStatus status, + string expectedError) + { + var (root, _, _, _, objects) = RootWithTwoItemLists(); + var runtime = new FakeRuntime(); + var probe = new RetailUiAutomationProbe(root, objects); + var logs = new List(); + string path = Path.Combine( + Path.GetTempPath(), + Path.GetRandomFileName() + ".ui-probe.txt"); + File.WriteAllLines(path, ["checkpoint stable", "input press CombatToggleCombat"]); + + try + { + using var runner = new RetailUiAutomationScriptRunner( + probe, + path, + dumpOnStart: false, + log: logs.Add, + runtime: runtime); + + runner.Tick(0d); + runtime.CompleteCheckpoint("stable", status, expectedError); + runner.Tick(0d); + + Assert.True(runner.Completed); + Assert.Equal(["stable"], runtime.Checkpoints); + Assert.Contains(logs, line => line.Contains( + expectedError, + StringComparison.Ordinal)); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void ScriptRunner_disposeCancelsPendingCheckpoint() + { + var (root, _, _, _, objects) = RootWithTwoItemLists(); + var runtime = new FakeRuntime(); + var probe = new RetailUiAutomationProbe(root, objects); + string path = Path.Combine( + Path.GetTempPath(), + Path.GetRandomFileName() + ".ui-probe.txt"); + File.WriteAllText(path, "checkpoint stable"); + + try + { + var runner = new RetailUiAutomationScriptRunner( + probe, + path, + dumpOnStart: false, + runtime: runtime); + runner.Tick(0d); + + runner.Dispose(); + + Assert.Equal(["stable"], runtime.Checkpoints); + Assert.Throws(() => + runtime.CompleteCheckpoint("stable")); + } + finally + { + File.Delete(path); + } + } + [Fact] public void ScriptRunner_worldReadyTimeout_stopsWithActionableLine() { diff --git a/tools/connected-r6-soak.route.txt b/tools/connected-r6-soak.route.txt index a99d417b..6a3e77c5 100644 --- a/tools/connected-r6-soak.route.txt +++ b/tools/connected-r6-soak.route.txt @@ -40,6 +40,7 @@ input up MovementForward sleep 700 input press CombatToggleCombat sleep 7400 +checkpoint caul-baseline # Sawato baseline. command /teleloc 0x3032001C 83.3 89.2 133.005 @@ -49,6 +50,7 @@ input down MovementTurnRight sleep 6000 input up MovementTurnRight sleep 22000 +checkpoint sawato-baseline # Rynthid world-edge streaming. command /teleloc 0xF6820033 145.7 49.855 58.010 @@ -58,6 +60,7 @@ input down MovementTurnRight sleep 6000 input up MovementTurnRight sleep 22000 +checkpoint rynthid # Aerlinthe dense island. command /teleloc 0x09040008 11.4 188.6 87.705 @@ -67,6 +70,7 @@ input down MovementTurnRight sleep 6000 input up MovementTurnRight sleep 22000 +checkpoint aerlinthe # Same-location lifecycle revisit. command /teleloc 0x3032001C 83.3 89.2 133.005 @@ -76,6 +80,7 @@ input down MovementTurnRight sleep 6000 input up MovementTurnRight sleep 22000 +checkpoint sawato-return # Holtburg mixed-town workload + local exercise. command /teleloc 0xA9B40019 84.0 7.1 94.005 @@ -113,6 +118,7 @@ input up MovementForward sleep 700 input press CombatToggleCombat sleep 7400 +checkpoint holtburg # Caul return/leak oracle + final local exercise. command /teleloc 0xC95B0001 14.8 0.3 12.005 @@ -150,6 +156,7 @@ input up MovementForward sleep 700 input press CombatToggleCombat sleep 15000 +checkpoint caul-return # A second warm-cache Sawato/Caul revisit is the plateau oracle. The first # circuit intentionally warms bounded DAT/GPU caches, so comparing the cold @@ -162,6 +169,7 @@ input down MovementTurnRight sleep 6000 input up MovementTurnRight sleep 22000 +checkpoint sawato-plateau command /teleloc 0xC95B0001 14.8 0.3 12.005 wait materialized 9 60000 @@ -170,3 +178,4 @@ input down MovementTurnRight sleep 6000 input up MovementTurnRight sleep 22000 +checkpoint caul-plateau diff --git a/tools/run-connected-r6-soak.ps1 b/tools/run-connected-r6-soak.ps1 index 460d0c56..5775f451 100644 --- a/tools/run-connected-r6-soak.ps1 +++ b/tools/run-connected-r6-soak.ps1 @@ -24,6 +24,17 @@ $destinations = @( [pscustomobject]@{ Name = 'Sawato plateau'; Occurrence = 8; Exercise = $false }, [pscustomobject]@{ Name = 'Caul plateau'; Occurrence = 9; Exercise = $false } ) +$expectedCheckpointNames = @( + 'caul-baseline', + 'sawato-baseline', + 'rynthid', + 'aerlinthe', + 'sawato-return', + 'holtburg', + 'caul-return', + 'sawato-plateau', + 'caul-plateau' +) $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' $logDir = Join-Path $Repository 'logs' @@ -35,6 +46,7 @@ $markerLog = "$prefix.markers.log" $sampleCsv = "$prefix.samples.csv" $reportJson = "$prefix.report.json" $artifactDir = "$prefix.artifacts" +$checkpointTimeline = Join-Path $artifactDir 'world-lifecycle.checkpoints.jsonl' $routePath = Join-Path $Repository 'tools\connected-r6-soak.route.txt' $exe = Join-Path $Repository 'src\AcDream.App\bin\Release\net10.0\AcDream.App.exe' @@ -45,6 +57,7 @@ $process = $null $stopwatch = [Diagnostics.Stopwatch]::StartNew() $gracefulExit = $false $exitCode = $null +$canonicalCheckpoints = @() function Read-LogLines([string]$Path) { if (-not (Test-Path -LiteralPath $Path)) { return @() } @@ -253,6 +266,203 @@ function Observe-R6Exercise( Write-Marker "exercise destination='$Destination' forward=$forwardPressDelta/$forwardReleaseDelta jump=$jumpPressDelta/$jumpReleaseDelta combat=$combatDelta moveTruth=$moveTruthDelta" } +function Read-CanonicalCheckpoints { + if (-not (Test-Path -LiteralPath $checkpointTimeline)) { return @() } + + $records = [System.Collections.Generic.List[object]]::new() + $lineNumber = 0 + foreach ($line in Get-Content -LiteralPath $checkpointTimeline) { + $lineNumber++ + if ([string]::IsNullOrWhiteSpace($line)) { continue } + try { + $records.Add(($line | ConvertFrom-Json)) + } + catch { + throw "checkpoint timeline line $lineNumber is invalid JSON: $($_.Exception.Message)" + } + } + return @($records) +} + +function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) { + if ($canonicalCheckpoints.Count -ne $expectedCheckpointNames.Count) { + $failures.Add("expected $($expectedCheckpointNames.Count) canonical checkpoints, found $($canonicalCheckpoints.Count)") + } + + $count = [Math]::Min($canonicalCheckpoints.Count, $expectedCheckpointNames.Count) + for ($index = 0; $index -lt $count; $index++) { + $checkpoint = $canonicalCheckpoints[$index] + $expectedName = $expectedCheckpointNames[$index] + $expectedSequence = $index + 1 + if ($checkpoint.name -ne $expectedName) { + $failures.Add("canonical checkpoint $expectedSequence was '$($checkpoint.name)', expected '$expectedName'") + } + if ([int]$checkpoint.sequence -ne $expectedSequence) { + $failures.Add("canonical checkpoint '$expectedName' had sequence $($checkpoint.sequence), expected $expectedSequence") + } + if ([int]$checkpoint.processId -ne $Client.Id) { + $failures.Add("canonical checkpoint '$expectedName' came from process $($checkpoint.processId), expected $($Client.Id)") + } + + if ($null -eq $checkpoint.render -or $null -eq $checkpoint.resources) { + $failures.Add("canonical checkpoint '$expectedName' is missing render/resources data") + continue + } + + $renderWorldProperty = $checkpoint.render.PSObject.Properties['world'] + $renderVisibleProperty = if ($null -ne $renderWorldProperty) { + $renderWorldProperty.Value.PSObject.Properties['visibleLandblocks'] + } + $renderTotalProperty = if ($null -ne $renderWorldProperty) { + $renderWorldProperty.Value.PSObject.Properties['totalLandblocks'] + } + $resourceVisibleProperty = + $checkpoint.resources.PSObject.Properties['visibleLandblocks'] + $resourceTotalProperty = + $checkpoint.resources.PSObject.Properties['totalLandblocks'] + if ($null -eq $renderWorldProperty -or + $null -eq $renderVisibleProperty -or + $null -eq $renderTotalProperty -or + $null -eq $resourceVisibleProperty -or + $null -eq $resourceTotalProperty) { + $failures.Add("canonical checkpoint '$expectedName' is missing render-world synchronization fields") + } + elseif ([int]$renderVisibleProperty.Value -ne + [int]$resourceVisibleProperty.Value) { + $failures.Add("canonical checkpoint '$expectedName' mixed render/resource visible-landblock frames") + } + elseif ([int]$renderTotalProperty.Value -ne + [int]$resourceTotalProperty.Value) { + $failures.Add("canonical checkpoint '$expectedName' mixed render/resource total-landblock frames") + } + + $namedCheckpointPath = + Join-Path $artifactDir "checkpoint-$expectedName.json" + if (-not (Test-Path -LiteralPath $namedCheckpointPath)) { + $failures.Add("canonical checkpoint '$expectedName' is missing named artifact '$namedCheckpointPath'") + } + else { + try { + $namedCheckpoint = + Get-Content -LiteralPath $namedCheckpointPath -Raw | + ConvertFrom-Json + if ($namedCheckpoint.name -ne $checkpoint.name -or + [int]$namedCheckpoint.sequence -ne [int]$checkpoint.sequence -or + [int]$namedCheckpoint.processId -ne [int]$checkpoint.processId) { + $failures.Add("canonical checkpoint '$expectedName' named artifact does not match its timeline row") + } + } + catch { + $failures.Add("canonical checkpoint '$expectedName' named artifact is invalid JSON: $($_.Exception.Message)") + } + } + + foreach ($field in @( + 'pendingLiveTeardowns', + 'pendingLandblockRetirements', + 'stagedMeshUploads', + 'stagedMeshBytes', + 'compositeWarmupPending')) { + $property = $checkpoint.resources.PSObject.Properties[$field] + if ($null -eq $property) { + $failures.Add("canonical checkpoint '$expectedName' is missing resources.$field") + } + elseif ([long]$property.Value -ne 0) { + $failures.Add("canonical checkpoint '$expectedName' has resources.$field=$($property.Value), expected zero") + } + } + } + + $caulReturn = $canonicalCheckpoints | + Where-Object { $_.name -eq 'caul-return' } | + Select-Object -First 1 + $caulPlateau = $canonicalCheckpoints | + Where-Object { $_.name -eq 'caul-plateau' } | + Select-Object -First 1 + if ($null -eq $caulReturn -or $null -eq $caulPlateau) { + $failures.Add('canonical Caul return/plateau comparison is unavailable') + return + } + + foreach ($field in @( + 'loadedLandblocks', + 'visibleLandblocks', + 'totalLandblocks', + 'pendingLiveTeardowns', + 'pendingLandblockRetirements', + 'meshRenderData', + 'meshAtlasArrays', + 'meshEstimatedBytes', + 'stagedMeshUploads', + 'stagedMeshBytes', + 'trackedGpuBytes', + 'trackedGpuBuffers', + 'trackedGpuTextures', + 'ownedCompositeTextures', + 'compositeTextureOwners', + 'activeParticleTextures', + 'particleTextureOwners', + 'compositeWarmupPending')) { + $first = $caulReturn.resources.PSObject.Properties[$field] + $plateau = $caulPlateau.resources.PSObject.Properties[$field] + if ($null -eq $first -or $null -eq $plateau) { + $failures.Add("canonical Caul comparison is missing resources.$field") + } + elseif ([long]$first.Value -ne [long]$plateau.Value) { + $failures.Add("Caul plateau canonical owner '$field' changed $($first.Value) -> $($plateau.Value)") + } + } + + $populationChanged = $false + foreach ($field in @( + 'worldEntities', + 'animatedEntities', + 'liveEntities', + 'materializedLiveEntities')) { + $first = $caulReturn.resources.PSObject.Properties[$field] + $plateau = $caulPlateau.resources.PSObject.Properties[$field] + if ($null -eq $first -or $null -eq $plateau) { + $failures.Add("canonical Caul population comparison is missing resources.$field") + } + elseif ([long]$first.Value -ne [long]$plateau.Value) { + $populationChanged = $true + $warnings.Add("Caul plateau authoritative population '$field' changed $($first.Value) -> $($plateau.Value)") + } + } + + foreach ($field in @( + 'particleBindings', + 'particleOwners', + 'effectOwners', + 'lightOwners', + 'scriptOwners')) { + $first = $caulReturn.resources.PSObject.Properties[$field] + $plateau = $caulPlateau.resources.PSObject.Properties[$field] + if ($null -eq $first -or $null -eq $plateau) { + $failures.Add("canonical Caul owner comparison is missing resources.$field") + } + elseif ([long]$first.Value -ne [long]$plateau.Value) { + $message = "Caul plateau population-sensitive owner '$field' changed $($first.Value) -> $($plateau.Value)" + if ($populationChanged) { $warnings.Add($message) } + else { $failures.Add($message) } + } + } + + foreach ($field in @( + 'particleEmitters', + 'particles', + 'activeScripts')) { + $first = $caulReturn.resources.PSObject.Properties[$field] + $plateau = $caulPlateau.resources.PSObject.Properties[$field] + if ($null -eq $first -or $null -eq $plateau) { + $failures.Add("canonical Caul transient comparison is missing resources.$field") + } + elseif ([long]$first.Value -ne [long]$plateau.Value) { + $warnings.Add("Caul plateau authoritative/transient workload '$field' changed $($first.Value) -> $($plateau.Value)") + } + } +} + function Add-RelativeGateFailures { $caulReturn = $samples | Where-Object { $_.Destination -eq 'Caul return' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 $caulPlateau = $samples | Where-Object { $_.Destination -eq 'Caul plateau' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 @@ -430,6 +640,14 @@ try { $failures.Add("expected $($destinations.Count) teleport materializations") } + $null = Wait-ForLogPattern ` + $process ` + $stdoutLog ` + '[world-gate] checkpoint name=caul-plateau sequence=9' ` + 1 ` + 30 + $canonicalCheckpoints = @(Read-CanonicalCheckpoints) + Add-CanonicalCheckpointFailures $process Add-RelativeGateFailures Write-Marker 'route-complete requesting graceful close' @@ -474,12 +692,14 @@ finally { SourceStatus = $sourceStatus VideoControllers = $videoControllers Samples = @($samples) + CanonicalCheckpoints = @($canonicalCheckpoints) Artifacts = [pscustomobject]@{ Stdout = $stdoutLog Stderr = $stderrLog Markers = $markerLog SamplesCsv = $sampleCsv ReportJson = $reportJson + CheckpointTimeline = $checkpointTimeline } } $report | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $reportJson -Encoding utf8