test(app): add canonical connected soak snapshots

Make scripted lifecycle checkpoints acknowledged post-diagnostics render barriers, capture the exact frame outcome beside canonical resource ownership, and harden the nine-stop route with ordered same-location cache and lifetime gates without weakening process residency thresholds.

Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Erik 2026-07-22 20:01:06 +02:00
parent 2862622ba2
commit bca4148739
17 changed files with 995 additions and 52 deletions

View file

@ -137,13 +137,16 @@ Every checkpoint hard-gates zero:
- composite warmup work. - composite warmup work.
The same-location Caul return → Caul plateau comparison requires exact equality The same-location Caul return → Caul plateau comparison requires exact equality
for deterministic owner/cache counts and bytes: loaded/visible/total landblocks, for deterministic cache/allocator state: loaded/visible/total landblocks,
materialized live owners, teardown/retirement queues, VFX binding/owner/script/ teardown/retirement queues, mesh render-data/atlas/estimated bytes, staged work,
light counts, mesh render-data/atlas/estimated bytes, staged work, tracked GPU tracked GPU bytes/buffers/textures, and composite/particle texture ownership.
bytes/buffers/textures, and composite/particle texture ownership. Connected Connected `WorldEntities`, `AnimatedEntities`, `LiveEntities`, and materialized
`WorldEntities`, `AnimatedEntities`, `LiveEntities`, particles, and active live entities may legitimately change with authoritative population; those
scripts may legitimately change with authoritative population or transient deltas are named workload warnings. VFX binding/owner/script/light counts remain
effects; those deltas remain named workload warnings, not silent exclusions. hard equality gates when that population is stable, and become named workload
warnings when the authoritative population changed. Particles, emitters, and
active scripts are always transient workload warnings rather than silent
exclusions.
Managed used/committed deltas are reported diagnostically. Existing Managed used/committed deltas are reported diagnostically. Existing
working/private-memory, update-p95, and allocation-p50 thresholds remain byte- working/private-memory, update-p95, and allocation-p50 thresholds remain byte-
for-byte unchanged as the secondary residency/performance guard. for-byte unchanged as the secondary residency/performance guard.

View file

@ -55,7 +55,6 @@ internal sealed record FrameRootDependencies(
internal sealed record FrameRootResult( internal sealed record FrameRootResult(
UpdateFrameOrchestrator Update, UpdateFrameOrchestrator Update,
RenderFrameOrchestrator Render, RenderFrameOrchestrator Render,
WorldLifecycleAutomationController? LifecycleAutomation,
FrameRootRuntimeBindings RuntimeBindings, FrameRootRuntimeBindings RuntimeBindings,
IDisposable FrameGraphPublication, IDisposable FrameGraphPublication,
AcDream.App.Net.LiveSessionHost SessionHost); AcDream.App.Net.LiveSessionHost SessionHost);
@ -420,7 +419,10 @@ internal sealed class FrameRootCompositionPhase
artifactDirectory, artifactDirectory,
message => d.Log("[UI-PROBE] " + message)); message => d.Log("[UI-PROBE] " + message));
bindings.Adopt( bindings.Adopt(
"world lifecycle automation", "world lifecycle automation owner",
lifecycleAutomation);
bindings.Adopt(
"world lifecycle automation binding",
interaction.LateBindings.Automation.Bind( interaction.LateBindings.Automation.Bind(
lifecycleAutomation)); lifecycleAutomation));
} }
@ -453,6 +455,8 @@ internal sealed class FrameRootCompositionPhase
worldSceneRenderer, worldSceneRenderer,
privatePresentation, privatePresentation,
live.FrameDiagnostics, live.FrameDiagnostics,
(IRenderFramePostDiagnosticsPhase?)lifecycleAutomation
?? NullRenderFramePostDiagnosticsPhase.Instance,
(IRenderFrameFailureRecovery?)settings.DevTools?.Presenter (IRenderFrameFailureRecovery?)settings.DevTools?.Presenter
?? NullRenderFrameFailureRecovery.Instance); ?? NullRenderFrameFailureRecovery.Instance);
Fault(FrameRootCompositionPoint.RenderRootCreated); Fault(FrameRootCompositionPoint.RenderRootCreated);
@ -509,7 +513,6 @@ internal sealed class FrameRootCompositionPhase
var result = new FrameRootResult( var result = new FrameRootResult(
updateFrame, updateFrame,
renderFrame, renderFrame,
lifecycleAutomation,
bindings, bindings,
frameGraphPublication, frameGraphPublication,
session.SessionHost); session.SessionHost);

View file

@ -446,14 +446,25 @@ internal sealed class DeferredWorldLifecycleAutomationRuntime
return new ExpectedOwnerBinding<IRetailUiAutomationRuntime>(target, Release); return new ExpectedOwnerBinding<IRetailUiAutomationRuntime>(target, Release);
} }
public bool TryWriteCheckpoint(string name, out string error) public bool TryRequestCheckpoint(
string name,
out IRetailUiAutomationCheckpoint? checkpoint,
out string error)
{ {
if (!_deactivated && _target is { } target) if (!_deactivated && _target is { } target)
return target.TryWriteCheckpoint(name, out error); return target.TryRequestCheckpoint(name, out checkpoint, out error);
checkpoint = null;
error = "world lifecycle automation is not bound"; error = "world lifecycle automation is not bound";
return false; return false;
} }
public void CancelCheckpoint(IRetailUiAutomationCheckpoint checkpoint)
{
ArgumentNullException.ThrowIfNull(checkpoint);
if (!_deactivated && _target is { } target)
target.CancelCheckpoint(checkpoint);
}
public bool TryRequestScreenshot(string name, out string error) public bool TryRequestScreenshot(string name, out string error)
{ {
if (!_deactivated && _target is { } target) if (!_deactivated && _target is { } target)

View file

@ -1,4 +1,5 @@
using System.Text.Json; using System.Text.Json;
using AcDream.App.Rendering;
using AcDream.App.Streaming; using AcDream.App.Streaming;
using AcDream.App.UI.Testing; using AcDream.App.UI.Testing;
@ -71,14 +72,73 @@ internal sealed record WorldLifecycleCheckpoint(
DateTime TimestampUtc, DateTime TimestampUtc,
int ProcessId, int ProcessId,
WorldRevealLifecycleSnapshot Reveal, WorldRevealLifecycleSnapshot Reveal,
RenderFrameOutcome Render,
WorldLifecycleResourceSnapshot Resources); WorldLifecycleResourceSnapshot Resources);
internal sealed class WorldLifecycleCheckpointRequest :
IRetailUiAutomationCheckpoint
{
private readonly object _sync = new();
private RetailUiAutomationCheckpointStatus _status =
RetailUiAutomationCheckpointStatus.Pending;
private string? _error;
public WorldLifecycleCheckpointRequest(object owner, int sequence, string name)
{
Owner = owner ?? throw new ArgumentNullException(nameof(owner));
Sequence = sequence;
Name = name ?? throw new ArgumentNullException(nameof(name));
}
internal object Owner { get; }
public int Sequence { get; }
public string Name { get; }
public RetailUiAutomationCheckpointStatus Status
{
get
{
lock (_sync)
return _status;
}
}
public string? Error
{
get
{
lock (_sync)
return _error;
}
}
internal bool TrySetTerminal(
RetailUiAutomationCheckpointStatus status,
string? error)
{
if (status == RetailUiAutomationCheckpointStatus.Pending)
throw new ArgumentOutOfRangeException(nameof(status));
lock (_sync)
{
if (_status != RetailUiAutomationCheckpointStatus.Pending)
return false;
_status = status;
_error = error;
return true;
}
}
}
/// <summary> /// <summary>
/// Diagnostic-only runtime seam used by production retained-UI scripts. It /// Diagnostic-only runtime seam used by production retained-UI scripts. It
/// writes one structured checkpoint at explicit script edges and delegates GL /// writes one structured checkpoint at explicit script edges and delegates GL
/// capture to the render-thread screenshot owner. /// capture to the render-thread screenshot owner.
/// </summary> /// </summary>
internal sealed class WorldLifecycleAutomationController : IRetailUiAutomationRuntime internal sealed class WorldLifecycleAutomationController :
IRetailUiAutomationRuntime,
IRenderFramePostDiagnosticsPhase,
IDisposable
{ {
private static readonly JsonSerializerOptions JsonOptions = new() private static readonly JsonSerializerOptions JsonOptions = new()
{ {
@ -87,16 +147,21 @@ internal sealed class WorldLifecycleAutomationController : IRetailUiAutomationRu
private readonly Func<WorldRevealLifecycleSnapshot> _getReveal; private readonly Func<WorldRevealLifecycleSnapshot> _getReveal;
private readonly Func<int> _getPortalMaterializationCount; private readonly Func<int> _getPortalMaterializationCount;
private readonly Func<WorldLifecycleResourceSnapshot> _captureResources; private readonly Func<RenderFrameOutcome, WorldLifecycleResourceSnapshot>
_captureResources;
private readonly FrameScreenshotController _screenshots; private readonly FrameScreenshotController _screenshots;
private readonly string _artifactDirectory; private readonly string _artifactDirectory;
private readonly Action<string> _log; private readonly Action<string> _log;
private readonly object _requestOwner = new();
private readonly object _sync = new();
private readonly Queue<WorldLifecycleCheckpointRequest> _requests = [];
private int _sequence; private int _sequence;
private bool _disposed;
public WorldLifecycleAutomationController( public WorldLifecycleAutomationController(
Func<WorldRevealLifecycleSnapshot> getReveal, Func<WorldRevealLifecycleSnapshot> getReveal,
Func<int> getPortalMaterializationCount, Func<int> getPortalMaterializationCount,
Func<WorldLifecycleResourceSnapshot> captureResources, Func<RenderFrameOutcome, WorldLifecycleResourceSnapshot> captureResources,
FrameScreenshotController screenshots, FrameScreenshotController screenshots,
string artifactDirectory, string artifactDirectory,
Action<string>? log = null) Action<string>? log = null)
@ -116,35 +181,127 @@ internal sealed class WorldLifecycleAutomationController : IRetailUiAutomationRu
public bool IsWorldViewportVisible => _getReveal().WorldViewportObserved; public bool IsWorldViewportVisible => _getReveal().WorldViewportObserved;
public int PortalMaterializationCount => _getPortalMaterializationCount(); public int PortalMaterializationCount => _getPortalMaterializationCount();
public bool TryWriteCheckpoint(string name, out string error) public bool TryRequestCheckpoint(
string name,
out IRetailUiAutomationCheckpoint? checkpoint,
out string error)
{ {
checkpoint = null;
if (!AutomationArtifactName.TryValidate(name, out error)) if (!AutomationArtifactName.TryValidate(name, out error))
return false; return false;
lock (_sync)
{
if (_disposed)
{
error = $"checkpoint '{name}' was rejected because world lifecycle automation is shutting down";
return false;
}
var request = new WorldLifecycleCheckpointRequest(
_requestOwner,
checked(++_sequence),
name);
_requests.Enqueue(request);
checkpoint = request;
error = string.Empty;
return true;
}
}
public void CancelCheckpoint(IRetailUiAutomationCheckpoint checkpoint)
{
ArgumentNullException.ThrowIfNull(checkpoint);
if (checkpoint is not WorldLifecycleCheckpointRequest request
|| !ReferenceEquals(request.Owner, _requestOwner))
{
throw new ArgumentException(
"The checkpoint acknowledgement is not owned by this automation runtime.",
nameof(checkpoint));
}
lock (_sync)
{
request.TrySetTerminal(
RetailUiAutomationCheckpointStatus.Cancelled,
$"checkpoint '{request.Name}' was cancelled before render-frame capture");
}
}
public void Process(RenderFrameInput input, RenderFrameOutcome outcome)
{
lock (_sync)
{
if (_disposed)
return;
while (_requests.Count != 0)
{
WorldLifecycleCheckpointRequest request = _requests.Dequeue();
if (request.Status != RetailUiAutomationCheckpointStatus.Pending)
continue;
WriteCheckpoint(request, outcome);
}
}
}
private void WriteCheckpoint(
WorldLifecycleCheckpointRequest request,
RenderFrameOutcome outcome)
{
try try
{ {
Directory.CreateDirectory(_artifactDirectory); Directory.CreateDirectory(_artifactDirectory);
var checkpoint = new WorldLifecycleCheckpoint( var checkpoint = new WorldLifecycleCheckpoint(
Sequence: checked(++_sequence), Sequence: request.Sequence,
Name: name, Name: request.Name,
TimestampUtc: DateTime.UtcNow, TimestampUtc: DateTime.UtcNow,
ProcessId: Environment.ProcessId, ProcessId: Environment.ProcessId,
Reveal: _getReveal(), Reveal: _getReveal(),
Resources: _captureResources()); Render: outcome,
Resources: _captureResources(outcome));
string json = JsonSerializer.Serialize(checkpoint, JsonOptions); string json = JsonSerializer.Serialize(checkpoint, JsonOptions);
string timelinePath = Path.Combine(_artifactDirectory, "world-lifecycle.checkpoints.jsonl"); string timelinePath = Path.Combine(
File.AppendAllText(timelinePath, json + Environment.NewLine); _artifactDirectory,
"world-lifecycle.checkpoints.jsonl");
File.WriteAllText( File.WriteAllText(
Path.Combine(_artifactDirectory, $"checkpoint-{name}.json"), Path.Combine(
_artifactDirectory,
$"checkpoint-{request.Name}.json"),
json + Environment.NewLine); json + Environment.NewLine);
_log($"[world-gate] checkpoint name={name} sequence={checkpoint.Sequence} path={timelinePath}"); // Timeline publication is the commit edge: a visible row always
error = string.Empty; // has its complete named checkpoint artifact beside it.
return true; File.AppendAllText(timelinePath, json + Environment.NewLine);
request.TrySetTerminal(
RetailUiAutomationCheckpointStatus.Succeeded,
error: null);
_log(
$"[world-gate] checkpoint name={request.Name} "
+ $"sequence={request.Sequence} path={timelinePath}");
} }
catch (Exception exception) catch (Exception exception)
{ {
error = $"checkpoint '{name}' failed: {exception.Message}"; request.TrySetTerminal(
return false; RetailUiAutomationCheckpointStatus.Failed,
$"checkpoint '{request.Name}' sequence {request.Sequence} failed: "
+ exception.Message);
}
}
public void Dispose()
{
lock (_sync)
{
if (_disposed)
return;
_disposed = true;
while (_requests.Count != 0)
{
WorldLifecycleCheckpointRequest request = _requests.Dequeue();
request.TrySetTerminal(
RetailUiAutomationCheckpointStatus.Cancelled,
$"checkpoint '{request.Name}' was cancelled during world lifecycle automation shutdown");
}
} }
} }

View file

@ -70,7 +70,7 @@ internal sealed class WorldLifecycleResourceSnapshotSource
?? throw new ArgumentNullException(nameof(frameProfiler)); ?? throw new ArgumentNullException(nameof(frameProfiler));
} }
public WorldLifecycleResourceSnapshot Capture() public WorldLifecycleResourceSnapshot Capture(RenderFrameOutcome outcome)
{ {
ObjectMeshManager meshManager = _meshes.MeshManager ObjectMeshManager meshManager = _meshes.MeshManager
?? throw new InvalidOperationException( ?? throw new InvalidOperationException(
@ -82,8 +82,8 @@ internal sealed class WorldLifecycleResourceSnapshotSource
LoadedLandblocks: _world.LoadedLandblockIds.Count, LoadedLandblocks: _world.LoadedLandblockIds.Count,
WorldEntities: _world.Entities.Count, WorldEntities: _world.Entities.Count,
AnimatedEntities: _animations.Count, AnimatedEntities: _animations.Count,
VisibleLandblocks: render.VisibleLandblocks, VisibleLandblocks: outcome.World.VisibleLandblocks,
TotalLandblocks: render.TotalLandblocks, TotalLandblocks: outcome.World.TotalLandblocks,
LiveEntities: _liveEntities.Count, LiveEntities: _liveEntities.Count,
MaterializedLiveEntities: _liveEntities.MaterializedCount, MaterializedLiveEntities: _liveEntities.MaterializedCount,
PendingLiveTeardowns: _liveEntities.PendingTeardownCount, PendingLiveTeardowns: _liveEntities.PendingTeardownCount,

View file

@ -103,7 +103,6 @@ public sealed class GameWindow :
private LivePresentationRuntimeBindings? _livePresentationBindings; private LivePresentationRuntimeBindings? _livePresentationBindings;
private SessionPlayerRuntimeBindings? _sessionPlayerBindings; private SessionPlayerRuntimeBindings? _sessionPlayerBindings;
private AcDream.App.Diagnostics.FrameScreenshotController? _frameScreenshots; private AcDream.App.Diagnostics.FrameScreenshotController? _frameScreenshots;
private AcDream.App.Diagnostics.WorldLifecycleAutomationController? _worldLifecycleAutomation;
private FrameRootRuntimeBindings? _frameRootBindings; private FrameRootRuntimeBindings? _frameRootBindings;
private IDisposable? _frameGraphPublication; private IDisposable? _frameGraphPublication;
private AcDream.App.Rendering.GpuFrameFlightController? _gpuFrameFlights; private AcDream.App.Rendering.GpuFrameFlightController? _gpuFrameFlights;
@ -1048,15 +1047,13 @@ public sealed class GameWindow :
FrameRootResult result) FrameRootResult result)
{ {
ArgumentNullException.ThrowIfNull(result); ArgumentNullException.ThrowIfNull(result);
if (_worldLifecycleAutomation is not null if (_frameRootBindings is not null
|| _frameRootBindings is not null
|| _frameGraphPublication is not null) || _frameGraphPublication is not null)
{ {
throw new InvalidOperationException( throw new InvalidOperationException(
"The GameWindow composition shell already owns frame roots."); "The GameWindow composition shell already owns frame roots.");
} }
_worldLifecycleAutomation = result.LifecycleAutomation;
_frameRootBindings = result.RuntimeBindings; _frameRootBindings = result.RuntimeBindings;
_frameGraphPublication = result.FrameGraphPublication; _frameGraphPublication = result.FrameGraphPublication;
} }

View file

@ -60,6 +60,25 @@ internal interface IRenderFrameDiagnosticsPhase
void Publish(RenderFrameInput input, RenderFrameOutcome outcome); void Publish(RenderFrameInput input, RenderFrameOutcome outcome);
} }
internal interface IRenderFramePostDiagnosticsPhase
{
void Process(RenderFrameInput input, RenderFrameOutcome outcome);
}
internal sealed class NullRenderFramePostDiagnosticsPhase :
IRenderFramePostDiagnosticsPhase
{
public static NullRenderFramePostDiagnosticsPhase Instance { get; } = new();
private NullRenderFramePostDiagnosticsPhase()
{
}
public void Process(RenderFrameInput input, RenderFrameOutcome outcome)
{
}
}
/// <summary> /// <summary>
/// Closes presentation transactions which may have opened before a later /// Closes presentation transactions which may have opened before a later
/// render phase failed. Recovery is idempotent so failures before the /// render phase failed. Recovery is idempotent so failures before the
@ -102,6 +121,7 @@ internal sealed class RenderFrameOrchestrator : IGameRenderFrameRoot
private readonly IWorldSceneFramePhase _world; private readonly IWorldSceneFramePhase _world;
private readonly IPrivatePresentationFramePhase _presentation; private readonly IPrivatePresentationFramePhase _presentation;
private readonly IRenderFrameDiagnosticsPhase _diagnostics; private readonly IRenderFrameDiagnosticsPhase _diagnostics;
private readonly IRenderFramePostDiagnosticsPhase _postDiagnostics;
private readonly IRenderFrameFailureRecovery _recovery; private readonly IRenderFrameFailureRecovery _recovery;
public RenderFrameOrchestrator( public RenderFrameOrchestrator(
@ -110,6 +130,7 @@ internal sealed class RenderFrameOrchestrator : IGameRenderFrameRoot
IWorldSceneFramePhase world, IWorldSceneFramePhase world,
IPrivatePresentationFramePhase presentation, IPrivatePresentationFramePhase presentation,
IRenderFrameDiagnosticsPhase diagnostics, IRenderFrameDiagnosticsPhase diagnostics,
IRenderFramePostDiagnosticsPhase postDiagnostics,
IRenderFrameFailureRecovery recovery) IRenderFrameFailureRecovery recovery)
{ {
_lifetime = lifetime ?? throw new ArgumentNullException(nameof(lifetime)); _lifetime = lifetime ?? throw new ArgumentNullException(nameof(lifetime));
@ -117,6 +138,8 @@ internal sealed class RenderFrameOrchestrator : IGameRenderFrameRoot
_world = world ?? throw new ArgumentNullException(nameof(world)); _world = world ?? throw new ArgumentNullException(nameof(world));
_presentation = presentation ?? throw new ArgumentNullException(nameof(presentation)); _presentation = presentation ?? throw new ArgumentNullException(nameof(presentation));
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
_postDiagnostics = postDiagnostics
?? throw new ArgumentNullException(nameof(postDiagnostics));
_recovery = recovery ?? throw new ArgumentNullException(nameof(recovery)); _recovery = recovery ?? throw new ArgumentNullException(nameof(recovery));
} }
@ -132,6 +155,7 @@ internal sealed class RenderFrameOrchestrator : IGameRenderFrameRoot
_presentation.Render(input, world); _presentation.Render(input, world);
var outcome = new RenderFrameOutcome(world, presentation); var outcome = new RenderFrameOutcome(world, presentation);
_diagnostics.Publish(input, outcome); _diagnostics.Publish(input, outcome);
_postDiagnostics.Process(input, outcome);
return outcome; return outcome;
} }
catch (Exception error) catch (Exception error)

View file

@ -7,6 +7,22 @@ using AcDream.UI.Abstractions.Input;
namespace AcDream.App.UI.Testing; namespace AcDream.App.UI.Testing;
public enum RetailUiAutomationCheckpointStatus
{
Pending,
Succeeded,
Failed,
Cancelled,
}
public interface IRetailUiAutomationCheckpoint
{
int Sequence { get; }
string Name { get; }
RetailUiAutomationCheckpointStatus Status { get; }
string? Error { get; }
}
/// <summary> /// <summary>
/// Narrow bridge from retained-UI scripts to render/world lifecycle /// Narrow bridge from retained-UI scripts to render/world lifecycle
/// diagnostics. Implementations run on the same update/render thread as the /// diagnostics. Implementations run on the same update/render thread as the
@ -17,7 +33,11 @@ public interface IRetailUiAutomationRuntime
bool IsWorldReady { get; } bool IsWorldReady { get; }
bool IsWorldViewportVisible { get; } bool IsWorldViewportVisible { get; }
int PortalMaterializationCount { get; } int PortalMaterializationCount { get; }
bool TryWriteCheckpoint(string name, out string error); bool TryRequestCheckpoint(
string name,
out IRetailUiAutomationCheckpoint? checkpoint,
out string error);
void CancelCheckpoint(IRetailUiAutomationCheckpoint checkpoint);
bool TryRequestScreenshot(string name, out string error); bool TryRequestScreenshot(string name, out string error);
bool IsScreenshotComplete(string name); bool IsScreenshotComplete(string name);
} }
@ -40,6 +60,8 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
private readonly HashSet<InputAction> _heldInputs = new(); private readonly HashSet<InputAction> _heldInputs = new();
private readonly bool _dumpOnStart; private readonly bool _dumpOnStart;
private readonly string? _loadError; private readonly string? _loadError;
private IRetailUiAutomationCheckpoint? _checkpoint;
private int _checkpointCommandIndex = -1;
private int _index; private int _index;
private int _activeIndex = -1; private int _activeIndex = -1;
private double _commandElapsedMs; private double _commandElapsedMs;
@ -405,8 +427,47 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
return Stop(command, "usage: checkpoint <name>"); return Stop(command, "usage: checkpoint <name>");
if (_runtime is null) if (_runtime is null)
return Stop(command, "world lifecycle automation is unavailable"); return Stop(command, "world lifecycle automation is unavailable");
return _runtime.TryWriteCheckpoint(command.Parts[1], out string error)
|| Stop(command, error); if (_checkpoint is null)
{
if (!_runtime.TryRequestCheckpoint(
command.Parts[1],
out IRetailUiAutomationCheckpoint? checkpoint,
out string error))
{
return Stop(command, error);
}
if (checkpoint is null)
{
return Stop(
command,
"world lifecycle automation accepted a checkpoint without returning its acknowledgement");
}
_checkpoint = checkpoint;
_checkpointCommandIndex = _index;
}
if (_checkpointCommandIndex != _index)
{
return Stop(
command,
"checkpoint acknowledgement belongs to a different script command");
}
RetailUiAutomationCheckpointStatus status = _checkpoint.Status;
if (status == RetailUiAutomationCheckpointStatus.Pending)
return false;
string? terminalError = _checkpoint.Error;
_checkpoint = null;
_checkpointCommandIndex = -1;
return status == RetailUiAutomationCheckpointStatus.Succeeded
|| Stop(
command,
terminalError
?? $"checkpoint '{command.Parts[1]}' ended with {status}");
} }
private bool DoScreenshot(ScriptCommand command) private bool DoScreenshot(ScriptCommand command)
@ -476,6 +537,12 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
public void Dispose() public void Dispose()
{ {
if (_disposed) return; if (_disposed) return;
if (_checkpoint is not null)
{
_runtime?.CancelCheckpoint(_checkpoint);
_checkpoint = null;
_checkpointCommandIndex = -1;
}
ReleaseHeldInputs(); ReleaseHeldInputs();
_disposed = true; _disposed = true;
} }

View file

@ -39,7 +39,11 @@ public sealed class FrameRootCompositionTests
source, source,
"new RenderFrameResourceController(", "new RenderFrameResourceController(",
"new WorldSceneRenderer(", "new WorldSceneRenderer(",
"new WorldLifecycleAutomationController(",
"\"world lifecycle automation owner\"",
"\"world lifecycle automation binding\"",
"new RenderFrameOrchestrator(", "new RenderFrameOrchestrator(",
"(IRenderFramePostDiagnosticsPhase?)lifecycleAutomation",
"new RetailLiveFrameCoordinator(", "new RetailLiveFrameCoordinator(",
"new UpdateFrameOrchestrator(", "new UpdateFrameOrchestrator(",
"d.FrameGraphs.PublishOwned(", "d.FrameGraphs.PublishOwned(",
@ -62,6 +66,7 @@ public sealed class FrameRootCompositionTests
Assert.DoesNotContain("new AcDream.App.Rendering.WorldSceneRenderer(", source); Assert.DoesNotContain("new AcDream.App.Rendering.WorldSceneRenderer(", source);
Assert.DoesNotContain("new AcDream.App.Update.UpdateFrameOrchestrator(", source); Assert.DoesNotContain("new AcDream.App.Update.UpdateFrameOrchestrator(", source);
Assert.DoesNotContain("CaptureWorldLifecycleResourceSnapshot", source); Assert.DoesNotContain("CaptureWorldLifecycleResourceSnapshot", source);
Assert.DoesNotContain("_worldLifecycleAutomation", source);
Assert.DoesNotContain("_frameGraphs.Publish(", source); Assert.DoesNotContain("_frameGraphs.Publish(", source);
} }
@ -83,6 +88,9 @@ public sealed class FrameRootCompositionTests
Assert.Contains("_liveEntities.PendingTeardownCount", snapshots); Assert.Contains("_liveEntities.PendingTeardownCount", snapshots);
Assert.Contains("GpuMemoryTracker.AllocatedBytes", snapshots); Assert.Contains("GpuMemoryTracker.AllocatedBytes", snapshots);
Assert.Contains("_frameProfiler.LastReport", snapshots); Assert.Contains("_frameProfiler.LastReport", snapshots);
Assert.Contains("Capture(RenderFrameOutcome outcome)", snapshots);
Assert.Contains("outcome.World.VisibleLandblocks", snapshots);
Assert.Contains("outcome.World.TotalLandblocks", snapshots);
} }
private sealed class RetryBinding( private sealed class RetryBinding(

View file

@ -110,13 +110,14 @@ public sealed class InteractionUiRuntimeSourcesTests
public void AutomationProxyIsInertUntilExactRuntimeBinds() public void AutomationProxyIsInertUntilExactRuntimeBinds()
{ {
var source = new DeferredWorldLifecycleAutomationRuntime(); var source = new DeferredWorldLifecycleAutomationRuntime();
Assert.False(source.TryWriteCheckpoint("early", out string earlyError)); Assert.False(source.TryRequestCheckpoint(
"early", out _, out string earlyError));
Assert.Contains("not bound", earlyError); Assert.Contains("not bound", earlyError);
var target = new AutomationRuntime(); var target = new AutomationRuntime();
IDisposable binding = source.Bind(target); IDisposable binding = source.Bind(target);
Assert.True(source.IsWorldReady); Assert.True(source.IsWorldReady);
Assert.True(source.TryWriteCheckpoint("ready", out _)); Assert.True(source.TryRequestCheckpoint("ready", out _, out _));
Assert.Equal("ready", target.LastCheckpoint); Assert.Equal("ready", target.LastCheckpoint);
binding.Dispose(); binding.Dispose();
@ -198,18 +199,35 @@ public sealed class InteractionUiRuntimeSourcesTests
private sealed class AutomationRuntime private sealed class AutomationRuntime
: AcDream.App.UI.Testing.IRetailUiAutomationRuntime : AcDream.App.UI.Testing.IRetailUiAutomationRuntime
{ {
private sealed record CompletedCheckpoint(int Sequence, string Name)
: AcDream.App.UI.Testing.IRetailUiAutomationCheckpoint
{
public AcDream.App.UI.Testing.RetailUiAutomationCheckpointStatus Status =>
AcDream.App.UI.Testing.RetailUiAutomationCheckpointStatus.Succeeded;
public string? Error => null;
}
public bool IsWorldReady => true; public bool IsWorldReady => true;
public bool IsWorldViewportVisible => true; public bool IsWorldViewportVisible => true;
public int PortalMaterializationCount => 2; public int PortalMaterializationCount => 2;
public string? LastCheckpoint { get; private set; } public string? LastCheckpoint { get; private set; }
public bool TryWriteCheckpoint(string name, out string error) public bool TryRequestCheckpoint(
string name,
out AcDream.App.UI.Testing.IRetailUiAutomationCheckpoint? checkpoint,
out string error)
{ {
LastCheckpoint = name; LastCheckpoint = name;
checkpoint = new CompletedCheckpoint(1, name);
error = string.Empty; error = string.Empty;
return true; return true;
} }
public void CancelCheckpoint(
AcDream.App.UI.Testing.IRetailUiAutomationCheckpoint checkpoint)
{
}
public bool TryRequestScreenshot(string name, out string error) public bool TryRequestScreenshot(string name, out string error)
{ {
error = string.Empty; error = string.Empty;

View file

@ -0,0 +1,133 @@
namespace AcDream.App.Tests.Diagnostics;
public sealed class ConnectedR6SoakContractTests
{
private static readonly string[] ExpectedCheckpointNames =
[
"caul-baseline",
"sawato-baseline",
"rynthid",
"aerlinthe",
"sawato-return",
"holtburg",
"caul-return",
"sawato-plateau",
"caul-plateau",
];
[Fact]
public void RouteContainsExactlyNineOrderedCheckpointBarriers()
{
string[] checkpoints = File.ReadAllLines(Path.Combine(
FindRepoRoot(),
"tools",
"connected-r6-soak.route.txt"))
.Select(static line => line.Trim())
.Where(static line => line.StartsWith(
"checkpoint ",
StringComparison.Ordinal))
.Select(static line => line["checkpoint ".Length..])
.ToArray();
Assert.Equal(ExpectedCheckpointNames, checkpoints);
}
[Fact]
public void GateConsumesCanonicalTimelineWithoutWeakeningResidencyFormulas()
{
string source = File.ReadAllText(Path.Combine(
FindRepoRoot(),
"tools",
"run-connected-r6-soak.ps1"));
AssertAppearsInOrder(
source,
"$expectedCheckpointNames = @(",
"'caul-baseline'",
"'sawato-baseline'",
"'rynthid'",
"'aerlinthe'",
"'sawato-return'",
"'holtburg'",
"'caul-return'",
"'sawato-plateau'",
"'caul-plateau'");
AssertAppearsInOrder(
source,
"$canonicalCheckpoints = @(Read-CanonicalCheckpoints)",
"Add-CanonicalCheckpointFailures $process",
"Add-RelativeGateFailures");
Assert.Contains("CanonicalCheckpoints = @($canonicalCheckpoints)", source);
Assert.Contains("CheckpointTimeline = $checkpointTimeline", source);
Assert.Contains(
"Join-Path $artifactDir \"checkpoint-$expectedName.json\"",
source,
StringComparison.Ordinal);
Assert.Contains(
"missing render-world synchronization fields",
source,
StringComparison.Ordinal);
string[] zeroFields =
[
"pendingLiveTeardowns",
"pendingLandblockRetirements",
"stagedMeshUploads",
"stagedMeshBytes",
"compositeWarmupPending",
];
foreach (string field in zeroFields)
Assert.Contains($"'{field}'", source, StringComparison.Ordinal);
Assert.Equal(1, CountOccurrences(
source,
"$privateLimit = [Math]::Max(192.0, $pair.First.PrivateMiB * 0.20)"));
Assert.Equal(1, CountOccurrences(
source,
"$workingLimit = [Math]::Max(192.0, $pair.First.WorkingSetMiB * 0.20)"));
Assert.Equal(1, CountOccurrences(
source,
"$updateLimit = $pair.First.UpdateP95Ms * 1.5 + 0.5"));
Assert.Equal(1, CountOccurrences(
source,
"$allocLimit = $pair.First.AllocP50Kb * 1.5 + 16.0"));
}
private static int CountOccurrences(string source, string value)
{
int count = 0;
int cursor = 0;
while ((cursor = source.IndexOf(value, cursor, StringComparison.Ordinal)) >= 0)
{
count++;
cursor += value.Length;
}
return count;
}
private static void AssertAppearsInOrder(string source, params string[] values)
{
int cursor = -1;
foreach (string value in values)
{
int next = source.IndexOf(value, cursor + 1, StringComparison.Ordinal);
Assert.True(next >= 0, $"Missing expected source fragment: {value}");
Assert.True(next > cursor, $"Out-of-order source fragment: {value}");
cursor = next;
}
}
private static string FindRepoRoot()
{
DirectoryInfo? directory = new(AppContext.BaseDirectory);
while (directory is not null)
{
if (File.Exists(Path.Combine(directory.FullName, "AcDream.slnx")))
return directory.FullName;
directory = directory.Parent;
}
throw new DirectoryNotFoundException("Could not find AcDream.slnx.");
}
}

View file

@ -1,12 +1,21 @@
using System.Text.Json; using System.Text.Json;
using AcDream.App.Diagnostics; using AcDream.App.Diagnostics;
using AcDream.App.Rendering;
using AcDream.App.Streaming; using AcDream.App.Streaming;
using AcDream.App.UI.Testing;
using SixLabors.ImageSharp; using SixLabors.ImageSharp;
namespace AcDream.App.Tests.Diagnostics; namespace AcDream.App.Tests.Diagnostics;
public sealed class WorldLifecycleAutomationControllerTests public sealed class WorldLifecycleAutomationControllerTests
{ {
private static readonly RenderFrameInput FrameInput = new(1d / 60d, 1280, 720);
private static readonly RenderFrameOutcome FrameOutcome = new(
new WorldRenderFrameOutcome(4, 17, NormalWorldDrawn: true),
new PrivatePresentationFrameOutcome(
PortalViewportDrawn: false,
ScreenshotCaptured: true));
[Fact] [Fact]
public void ScreenshotCapture_FlipsGlRowsAndWritesACompletePng() public void ScreenshotCapture_FlipsGlRowsAndWritesACompletePng()
{ {
@ -112,7 +121,7 @@ public sealed class WorldLifecycleAutomationControllerTests
var controller = new WorldLifecycleAutomationController( var controller = new WorldLifecycleAutomationController(
() => reveal, () => reveal,
() => 3, () => 3,
() => resources, _ => resources,
screenshots, screenshots,
directory); directory);
@ -121,13 +130,26 @@ public sealed class WorldLifecycleAutomationControllerTests
Assert.True(controller.IsWorldReady); Assert.True(controller.IsWorldReady);
Assert.True(controller.IsWorldViewportVisible); Assert.True(controller.IsWorldViewportVisible);
Assert.Equal(3, controller.PortalMaterializationCount); Assert.Equal(3, controller.PortalMaterializationCount);
Assert.True(controller.TryWriteCheckpoint("dungeon", out string error), error); Assert.True(controller.TryRequestCheckpoint(
"dungeon",
out IRetailUiAutomationCheckpoint? request,
out string error), error);
Assert.NotNull(request);
Assert.Equal(RetailUiAutomationCheckpointStatus.Pending, request.Status);
controller.Process(FrameInput, FrameOutcome);
Assert.Equal(RetailUiAutomationCheckpointStatus.Succeeded, request.Status);
string line = Assert.Single(File.ReadAllLines( string line = Assert.Single(File.ReadAllLines(
Path.Combine(directory, "world-lifecycle.checkpoints.jsonl"))); Path.Combine(directory, "world-lifecycle.checkpoints.jsonl")));
using JsonDocument json = JsonDocument.Parse(line); using JsonDocument json = JsonDocument.Parse(line);
Assert.Equal("dungeon", json.RootElement.GetProperty("name").GetString()); Assert.Equal("dungeon", json.RootElement.GetProperty("name").GetString());
Assert.Equal(7, json.RootElement.GetProperty("reveal").GetProperty("generation").GetInt64()); Assert.Equal(7, json.RootElement.GetProperty("reveal").GetProperty("generation").GetInt64());
Assert.Equal(4, json.RootElement.GetProperty("render").GetProperty("world")
.GetProperty("visibleLandblocks").GetInt32());
Assert.True(json.RootElement.GetProperty("render").GetProperty("presentation")
.GetProperty("screenshotCaptured").GetBoolean());
Assert.Equal(42, json.RootElement.GetProperty("resources").GetProperty("worldEntities").GetInt32()); Assert.Equal(42, json.RootElement.GetProperty("resources").GetProperty("worldEntities").GetInt32());
Assert.True(File.Exists(Path.Combine(directory, "checkpoint-dungeon.json"))); Assert.True(File.Exists(Path.Combine(directory, "checkpoint-dungeon.json")));
} }
@ -137,6 +159,123 @@ public sealed class WorldLifecycleAutomationControllerTests
} }
} }
[Fact]
public void CheckpointRequests_DrainInFifoOrderOnce()
{
string directory = NewDirectory();
int captures = 0;
var controller = CreateController(
directory,
_ =>
{
captures++;
return EmptyResources();
});
try
{
Assert.True(controller.TryRequestCheckpoint(
"first", out IRetailUiAutomationCheckpoint? first, out _));
Assert.True(controller.TryRequestCheckpoint(
"second", out IRetailUiAutomationCheckpoint? second, out _));
controller.Process(FrameInput, FrameOutcome);
controller.Process(FrameInput, FrameOutcome);
Assert.NotNull(first);
Assert.NotNull(second);
Assert.Equal(1, first.Sequence);
Assert.Equal(2, second.Sequence);
Assert.Equal(RetailUiAutomationCheckpointStatus.Succeeded, first.Status);
Assert.Equal(RetailUiAutomationCheckpointStatus.Succeeded, second.Status);
Assert.Equal(2, captures);
string[] lines = File.ReadAllLines(
Path.Combine(directory, "world-lifecycle.checkpoints.jsonl"));
Assert.Equal(2, lines.Length);
Assert.Contains("\"name\":\"first\"", lines[0], StringComparison.Ordinal);
Assert.Contains("\"name\":\"second\"", lines[1], StringComparison.Ordinal);
}
finally
{
controller.Dispose();
Directory.Delete(directory, recursive: true);
}
}
[Fact]
public void CheckpointWriteFailure_IsReportedOnTokenWithoutThrowingFromFrame()
{
string directory = NewDirectory();
string blockedPath = Path.Combine(directory, "not-a-directory");
File.WriteAllText(blockedPath, "occupied");
var controller = CreateController(blockedPath, _ => EmptyResources());
try
{
Assert.True(controller.TryRequestCheckpoint(
"failed", out IRetailUiAutomationCheckpoint? request, out _));
controller.Process(FrameInput, FrameOutcome);
Assert.NotNull(request);
Assert.Equal(RetailUiAutomationCheckpointStatus.Failed, request.Status);
Assert.Contains("checkpoint 'failed' sequence 1 failed", request.Error);
}
finally
{
controller.Dispose();
Directory.Delete(directory, recursive: true);
}
}
[Fact]
public void CheckpointCancellationAndShutdown_AreExplicitAndNeverWritten()
{
string directory = NewDirectory();
var controller = CreateController(directory, _ => EmptyResources());
try
{
Assert.True(controller.TryRequestCheckpoint(
"caller", out IRetailUiAutomationCheckpoint? caller, out _));
Assert.True(controller.TryRequestCheckpoint(
"shutdown", out IRetailUiAutomationCheckpoint? shutdown, out _));
Assert.NotNull(caller);
Assert.NotNull(shutdown);
controller.CancelCheckpoint(caller);
controller.Dispose();
controller.Process(FrameInput, FrameOutcome);
Assert.Equal(RetailUiAutomationCheckpointStatus.Cancelled, caller.Status);
Assert.Contains("cancelled before render-frame capture", caller.Error);
Assert.Equal(RetailUiAutomationCheckpointStatus.Cancelled, shutdown.Status);
Assert.Contains("cancelled during world lifecycle automation shutdown", shutdown.Error);
Assert.False(controller.TryRequestCheckpoint(
"late", out IRetailUiAutomationCheckpoint? late, out string error));
Assert.Null(late);
Assert.Contains("shutting down", error);
Assert.False(File.Exists(Path.Combine(
directory,
"world-lifecycle.checkpoints.jsonl")));
}
finally
{
controller.Dispose();
Directory.Delete(directory, recursive: true);
}
}
private static WorldLifecycleAutomationController CreateController(
string directory,
Func<RenderFrameOutcome, WorldLifecycleResourceSnapshot> capture) =>
new(
() => default,
() => 0,
capture,
new FrameScreenshotController((_, _) => [], directory),
directory);
private static WorldLifecycleResourceSnapshot EmptyResources() => new( private static WorldLifecycleResourceSnapshot EmptyResources() => new(
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0L, 0, 0L, 0L, 0, 0, 0, 0, 0, 0, 0, 0L, 0L, 0d, 0d, null); 0, 0, 0L, 0, 0L, 0L, 0, 0, 0, 0, 0, 0, 0, 0L, 0L, 0d, 0d, null);

View file

@ -38,7 +38,10 @@ public sealed class RenderFrameOrchestratorTests
RenderFrameOutcome outcome = orchestrator.Render(Input); RenderFrameOutcome outcome = orchestrator.Render(Input);
Assert.Equal( Assert.Equal(
["gpu-begin", "resources", "world", "presentation", "diagnostics", "gpu-end"], [
"gpu-begin", "resources", "world", "presentation",
"diagnostics", "post-diagnostics", "gpu-end",
],
calls); calls);
Assert.Equal(phases.World, outcome.World); Assert.Equal(phases.World, outcome.World);
Assert.Equal(phases.Presentation, outcome.Presentation); Assert.Equal(phases.Presentation, outcome.Presentation);
@ -48,6 +51,7 @@ public sealed class RenderFrameOrchestratorTests
("world", Input), ("world", Input),
("presentation", Input), ("presentation", Input),
("diagnostics", Input), ("diagnostics", Input),
("post-diagnostics", Input),
], ],
phases.ObservedInputs); phases.ObservedInputs);
Assert.Equal(phases.World, phases.ObservedWorld); Assert.Equal(phases.World, phases.ObservedWorld);
@ -77,6 +81,7 @@ public sealed class RenderFrameOrchestratorTests
[InlineData("world")] [InlineData("world")]
[InlineData("presentation")] [InlineData("presentation")]
[InlineData("diagnostics")] [InlineData("diagnostics")]
[InlineData("post-diagnostics")]
public void RenderFailure_ClosesExactlyOnceAndPropagates(string failurePoint) public void RenderFailure_ClosesExactlyOnceAndPropagates(string failurePoint)
{ {
var calls = new List<string>(); var calls = new List<string>();
@ -112,7 +117,10 @@ public sealed class RenderFrameOrchestratorTests
Assert.Same(expected, actual); Assert.Same(expected, actual);
Assert.Equal( Assert.Equal(
["gpu-begin", "resources", "world", "presentation", "diagnostics", "gpu-end"], [
"gpu-begin", "resources", "world", "presentation",
"diagnostics", "post-diagnostics", "gpu-end",
],
calls); calls);
} }
@ -121,6 +129,7 @@ public sealed class RenderFrameOrchestratorTests
[InlineData("world")] [InlineData("world")]
[InlineData("presentation")] [InlineData("presentation")]
[InlineData("diagnostics")] [InlineData("diagnostics")]
[InlineData("post-diagnostics")]
public void RenderAndCloseFailure_AreAggregatedInCausalOrder(string failurePoint) public void RenderAndCloseFailure_AreAggregatedInCausalOrder(string failurePoint)
{ {
var calls = new List<string>(); var calls = new List<string>();
@ -208,17 +217,19 @@ public sealed class RenderFrameOrchestratorTests
var phases = new RecordingPhases([]); var phases = new RecordingPhases([]);
Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator( Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator(
null!, phases, phases, phases, phases, phases)); null!, phases, phases, phases, phases, phases, phases));
Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator( Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator(
phases, null!, phases, phases, phases, phases)); phases, null!, phases, phases, phases, phases, phases));
Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator( Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator(
phases, phases, null!, phases, phases, phases)); phases, phases, null!, phases, phases, phases, phases));
Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator( Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator(
phases, phases, phases, null!, phases, phases)); phases, phases, phases, null!, phases, phases, phases));
Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator( Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator(
phases, phases, phases, phases, null!, phases)); phases, phases, phases, phases, null!, phases, phases));
Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator( Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator(
phases, phases, phases, phases, phases, null!)); phases, phases, phases, phases, phases, null!, phases));
Assert.Throws<ArgumentNullException>(() => new RenderFrameOrchestrator(
phases, phases, phases, phases, phases, phases, null!));
} }
[Fact] [Fact]
@ -231,6 +242,7 @@ public sealed class RenderFrameOrchestratorTests
typeof(IWorldSceneFramePhase), typeof(IWorldSceneFramePhase),
typeof(IPrivatePresentationFramePhase), typeof(IPrivatePresentationFramePhase),
typeof(IRenderFrameDiagnosticsPhase), typeof(IRenderFrameDiagnosticsPhase),
typeof(IRenderFramePostDiagnosticsPhase),
typeof(IRenderFrameFailureRecovery), typeof(IRenderFrameFailureRecovery),
]; ];
FieldInfo[] fields = typeof(RenderFrameOrchestrator).GetFields( FieldInfo[] fields = typeof(RenderFrameOrchestrator).GetFields(
@ -308,6 +320,11 @@ public sealed class RenderFrameOrchestratorTests
"gpu-begin", "resources", "world", "presentation", "diagnostics", "gpu-begin", "resources", "world", "presentation", "diagnostics",
"recovery-abort", "gpu-end", "recovery-abort", "gpu-end",
], ],
"post-diagnostics" =>
[
"gpu-begin", "resources", "world", "presentation", "diagnostics",
"post-diagnostics", "recovery-abort", "gpu-end",
],
_ => throw new ArgumentOutOfRangeException(nameof(failurePoint)), _ => throw new ArgumentOutOfRangeException(nameof(failurePoint)),
}; };
@ -320,13 +337,18 @@ public sealed class RenderFrameOrchestratorTests
"world" => ["resources", "world"], "world" => ["resources", "world"],
"presentation" => ["resources", "world", "presentation"], "presentation" => ["resources", "world", "presentation"],
"diagnostics" => ["resources", "world", "presentation", "diagnostics"], "diagnostics" => ["resources", "world", "presentation", "diagnostics"],
"post-diagnostics" =>
[
"resources", "world", "presentation", "diagnostics",
"post-diagnostics",
],
_ => throw new ArgumentOutOfRangeException(nameof(failurePoint)), _ => throw new ArgumentOutOfRangeException(nameof(failurePoint)),
}; };
return phases.Select(phase => (phase, Input)).ToArray(); return phases.Select(phase => (phase, Input)).ToArray();
} }
private static RenderFrameOrchestrator Create(RecordingPhases phases) => private static RenderFrameOrchestrator Create(RecordingPhases phases) =>
new(phases, phases, phases, phases, phases, phases); new(phases, phases, phases, phases, phases, phases, phases);
private sealed class RecordingPhases : private sealed class RecordingPhases :
IRenderFrameLifetime, IRenderFrameLifetime,
@ -334,6 +356,7 @@ public sealed class RenderFrameOrchestratorTests
IWorldSceneFramePhase, IWorldSceneFramePhase,
IPrivatePresentationFramePhase, IPrivatePresentationFramePhase,
IRenderFrameDiagnosticsPhase, IRenderFrameDiagnosticsPhase,
IRenderFramePostDiagnosticsPhase,
IRenderFrameFailureRecovery IRenderFrameFailureRecovery
{ {
private readonly List<string> _calls; private readonly List<string> _calls;
@ -399,6 +422,13 @@ public sealed class RenderFrameOrchestratorTests
Record("diagnostics"); Record("diagnostics");
} }
public void Process(RenderFrameInput input, RenderFrameOutcome outcome)
{
ObservedInputs.Add(("post-diagnostics", input));
ObservedOutcome = outcome;
Record("post-diagnostics");
}
private void Record(string call) private void Record(string call)
{ {
_calls.Add(call); _calls.Add(call);

View file

@ -37,6 +37,7 @@ public sealed class RenderFrameRecoveryIntegrationTests
new World(failure), new World(failure),
presentation, presentation,
new Diagnostics(), new Diagnostics(),
NullRenderFramePostDiagnosticsPhase.Instance,
devTools); devTools);
Assert.Throws<InvalidOperationException>(() => orchestrator.Render(Input)); Assert.Throws<InvalidOperationException>(() => orchestrator.Render(Input));

View file

@ -26,6 +26,18 @@ public sealed class RetailUiAutomationProbeTests
private sealed class FakeRuntime : IRetailUiAutomationRuntime private sealed class FakeRuntime : IRetailUiAutomationRuntime
{ {
private sealed class FakeCheckpoint(int sequence, string name) :
IRetailUiAutomationCheckpoint
{
public int Sequence { get; } = sequence;
public string Name { get; } = name;
public RetailUiAutomationCheckpointStatus Status { get; set; } =
RetailUiAutomationCheckpointStatus.Pending;
public string? Error { get; set; }
}
private readonly List<FakeCheckpoint> _checkpointTokens = [];
public bool IsWorldReady { get; set; } public bool IsWorldReady { get; set; }
public bool IsWorldViewportVisible { get; set; } public bool IsWorldViewportVisible { get; set; }
public int PortalMaterializationCount { get; set; } public int PortalMaterializationCount { get; set; }
@ -33,13 +45,41 @@ public sealed class RetailUiAutomationProbeTests
public HashSet<string> ScreenshotRequests { get; } = new(); public HashSet<string> ScreenshotRequests { get; } = new();
public HashSet<string> CompletedScreenshots { get; } = new(); public HashSet<string> CompletedScreenshots { get; } = new();
public bool TryWriteCheckpoint(string name, out string error) public bool TryRequestCheckpoint(
string name,
out IRetailUiAutomationCheckpoint? checkpoint,
out string error)
{ {
Checkpoints.Add(name); Checkpoints.Add(name);
var token = new FakeCheckpoint(_checkpointTokens.Count + 1, name);
_checkpointTokens.Add(token);
checkpoint = token;
error = string.Empty; error = string.Empty;
return true; return true;
} }
public void CancelCheckpoint(IRetailUiAutomationCheckpoint checkpoint)
{
var token = Assert.IsType<FakeCheckpoint>(checkpoint);
token.Status = RetailUiAutomationCheckpointStatus.Cancelled;
token.Error = $"checkpoint '{token.Name}' cancelled";
}
public void CompleteCheckpoint(
string name,
RetailUiAutomationCheckpointStatus status =
RetailUiAutomationCheckpointStatus.Succeeded,
string? error = null)
{
FakeCheckpoint token = Assert.Single(
_checkpointTokens,
value => value.Name == name);
if (token.Status != RetailUiAutomationCheckpointStatus.Pending)
throw new InvalidOperationException("checkpoint is already terminal");
token.Status = status;
token.Error = error;
}
public bool TryRequestScreenshot(string name, out string error) public bool TryRequestScreenshot(string name, out string error)
{ {
ScreenshotRequests.Add(name); ScreenshotRequests.Add(name);
@ -590,6 +630,16 @@ public sealed class RetailUiAutomationProbeTests
runner.Tick(0.001d); runner.Tick(0.001d);
Assert.Equal(["stable"], runtime.Checkpoints); Assert.Equal(["stable"], runtime.Checkpoints);
Assert.DoesNotContain("stable", runtime.ScreenshotRequests);
Assert.False(runner.Completed);
runner.Tick(0.001d);
Assert.Equal(["stable"], runtime.Checkpoints);
Assert.DoesNotContain("stable", runtime.ScreenshotRequests);
runtime.CompleteCheckpoint("stable");
runner.Tick(0.001d);
Assert.Contains("stable", runtime.ScreenshotRequests); Assert.Contains("stable", runtime.ScreenshotRequests);
Assert.False(runner.Completed); Assert.False(runner.Completed);
@ -604,6 +654,79 @@ public sealed class RetailUiAutomationProbeTests
} }
} }
[Theory]
[InlineData(RetailUiAutomationCheckpointStatus.Failed, "deferred write failed")]
[InlineData(RetailUiAutomationCheckpointStatus.Cancelled, "shutdown cancelled")]
public void ScriptRunner_checkpointTerminalFailure_StopsWithExactError(
RetailUiAutomationCheckpointStatus status,
string expectedError)
{
var (root, _, _, _, objects) = RootWithTwoItemLists();
var runtime = new FakeRuntime();
var probe = new RetailUiAutomationProbe(root, objects);
var logs = new List<string>();
string path = Path.Combine(
Path.GetTempPath(),
Path.GetRandomFileName() + ".ui-probe.txt");
File.WriteAllLines(path, ["checkpoint stable", "input press CombatToggleCombat"]);
try
{
using var runner = new RetailUiAutomationScriptRunner(
probe,
path,
dumpOnStart: false,
log: logs.Add,
runtime: runtime);
runner.Tick(0d);
runtime.CompleteCheckpoint("stable", status, expectedError);
runner.Tick(0d);
Assert.True(runner.Completed);
Assert.Equal(["stable"], runtime.Checkpoints);
Assert.Contains(logs, line => line.Contains(
expectedError,
StringComparison.Ordinal));
}
finally
{
File.Delete(path);
}
}
[Fact]
public void ScriptRunner_disposeCancelsPendingCheckpoint()
{
var (root, _, _, _, objects) = RootWithTwoItemLists();
var runtime = new FakeRuntime();
var probe = new RetailUiAutomationProbe(root, objects);
string path = Path.Combine(
Path.GetTempPath(),
Path.GetRandomFileName() + ".ui-probe.txt");
File.WriteAllText(path, "checkpoint stable");
try
{
var runner = new RetailUiAutomationScriptRunner(
probe,
path,
dumpOnStart: false,
runtime: runtime);
runner.Tick(0d);
runner.Dispose();
Assert.Equal(["stable"], runtime.Checkpoints);
Assert.Throws<InvalidOperationException>(() =>
runtime.CompleteCheckpoint("stable"));
}
finally
{
File.Delete(path);
}
}
[Fact] [Fact]
public void ScriptRunner_worldReadyTimeout_stopsWithActionableLine() public void ScriptRunner_worldReadyTimeout_stopsWithActionableLine()
{ {

View file

@ -40,6 +40,7 @@ input up MovementForward
sleep 700 sleep 700
input press CombatToggleCombat input press CombatToggleCombat
sleep 7400 sleep 7400
checkpoint caul-baseline
# Sawato baseline. # Sawato baseline.
command /teleloc 0x3032001C 83.3 89.2 133.005 command /teleloc 0x3032001C 83.3 89.2 133.005
@ -49,6 +50,7 @@ input down MovementTurnRight
sleep 6000 sleep 6000
input up MovementTurnRight input up MovementTurnRight
sleep 22000 sleep 22000
checkpoint sawato-baseline
# Rynthid world-edge streaming. # Rynthid world-edge streaming.
command /teleloc 0xF6820033 145.7 49.855 58.010 command /teleloc 0xF6820033 145.7 49.855 58.010
@ -58,6 +60,7 @@ input down MovementTurnRight
sleep 6000 sleep 6000
input up MovementTurnRight input up MovementTurnRight
sleep 22000 sleep 22000
checkpoint rynthid
# Aerlinthe dense island. # Aerlinthe dense island.
command /teleloc 0x09040008 11.4 188.6 87.705 command /teleloc 0x09040008 11.4 188.6 87.705
@ -67,6 +70,7 @@ input down MovementTurnRight
sleep 6000 sleep 6000
input up MovementTurnRight input up MovementTurnRight
sleep 22000 sleep 22000
checkpoint aerlinthe
# Same-location lifecycle revisit. # Same-location lifecycle revisit.
command /teleloc 0x3032001C 83.3 89.2 133.005 command /teleloc 0x3032001C 83.3 89.2 133.005
@ -76,6 +80,7 @@ input down MovementTurnRight
sleep 6000 sleep 6000
input up MovementTurnRight input up MovementTurnRight
sleep 22000 sleep 22000
checkpoint sawato-return
# Holtburg mixed-town workload + local exercise. # Holtburg mixed-town workload + local exercise.
command /teleloc 0xA9B40019 84.0 7.1 94.005 command /teleloc 0xA9B40019 84.0 7.1 94.005
@ -113,6 +118,7 @@ input up MovementForward
sleep 700 sleep 700
input press CombatToggleCombat input press CombatToggleCombat
sleep 7400 sleep 7400
checkpoint holtburg
# Caul return/leak oracle + final local exercise. # Caul return/leak oracle + final local exercise.
command /teleloc 0xC95B0001 14.8 0.3 12.005 command /teleloc 0xC95B0001 14.8 0.3 12.005
@ -150,6 +156,7 @@ input up MovementForward
sleep 700 sleep 700
input press CombatToggleCombat input press CombatToggleCombat
sleep 15000 sleep 15000
checkpoint caul-return
# A second warm-cache Sawato/Caul revisit is the plateau oracle. The first # A second warm-cache Sawato/Caul revisit is the plateau oracle. The first
# circuit intentionally warms bounded DAT/GPU caches, so comparing the cold # circuit intentionally warms bounded DAT/GPU caches, so comparing the cold
@ -162,6 +169,7 @@ input down MovementTurnRight
sleep 6000 sleep 6000
input up MovementTurnRight input up MovementTurnRight
sleep 22000 sleep 22000
checkpoint sawato-plateau
command /teleloc 0xC95B0001 14.8 0.3 12.005 command /teleloc 0xC95B0001 14.8 0.3 12.005
wait materialized 9 60000 wait materialized 9 60000
@ -170,3 +178,4 @@ input down MovementTurnRight
sleep 6000 sleep 6000
input up MovementTurnRight input up MovementTurnRight
sleep 22000 sleep 22000
checkpoint caul-plateau

View file

@ -24,6 +24,17 @@ $destinations = @(
[pscustomobject]@{ Name = 'Sawato plateau'; Occurrence = 8; Exercise = $false }, [pscustomobject]@{ Name = 'Sawato plateau'; Occurrence = 8; Exercise = $false },
[pscustomobject]@{ Name = 'Caul plateau'; Occurrence = 9; Exercise = $false } [pscustomobject]@{ Name = 'Caul plateau'; Occurrence = 9; Exercise = $false }
) )
$expectedCheckpointNames = @(
'caul-baseline',
'sawato-baseline',
'rynthid',
'aerlinthe',
'sawato-return',
'holtburg',
'caul-return',
'sawato-plateau',
'caul-plateau'
)
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss' $stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
$logDir = Join-Path $Repository 'logs' $logDir = Join-Path $Repository 'logs'
@ -35,6 +46,7 @@ $markerLog = "$prefix.markers.log"
$sampleCsv = "$prefix.samples.csv" $sampleCsv = "$prefix.samples.csv"
$reportJson = "$prefix.report.json" $reportJson = "$prefix.report.json"
$artifactDir = "$prefix.artifacts" $artifactDir = "$prefix.artifacts"
$checkpointTimeline = Join-Path $artifactDir 'world-lifecycle.checkpoints.jsonl'
$routePath = Join-Path $Repository 'tools\connected-r6-soak.route.txt' $routePath = Join-Path $Repository 'tools\connected-r6-soak.route.txt'
$exe = Join-Path $Repository 'src\AcDream.App\bin\Release\net10.0\AcDream.App.exe' $exe = Join-Path $Repository 'src\AcDream.App\bin\Release\net10.0\AcDream.App.exe'
@ -45,6 +57,7 @@ $process = $null
$stopwatch = [Diagnostics.Stopwatch]::StartNew() $stopwatch = [Diagnostics.Stopwatch]::StartNew()
$gracefulExit = $false $gracefulExit = $false
$exitCode = $null $exitCode = $null
$canonicalCheckpoints = @()
function Read-LogLines([string]$Path) { function Read-LogLines([string]$Path) {
if (-not (Test-Path -LiteralPath $Path)) { return @() } if (-not (Test-Path -LiteralPath $Path)) { return @() }
@ -253,6 +266,203 @@ function Observe-R6Exercise(
Write-Marker "exercise destination='$Destination' forward=$forwardPressDelta/$forwardReleaseDelta jump=$jumpPressDelta/$jumpReleaseDelta combat=$combatDelta moveTruth=$moveTruthDelta" Write-Marker "exercise destination='$Destination' forward=$forwardPressDelta/$forwardReleaseDelta jump=$jumpPressDelta/$jumpReleaseDelta combat=$combatDelta moveTruth=$moveTruthDelta"
} }
function Read-CanonicalCheckpoints {
if (-not (Test-Path -LiteralPath $checkpointTimeline)) { return @() }
$records = [System.Collections.Generic.List[object]]::new()
$lineNumber = 0
foreach ($line in Get-Content -LiteralPath $checkpointTimeline) {
$lineNumber++
if ([string]::IsNullOrWhiteSpace($line)) { continue }
try {
$records.Add(($line | ConvertFrom-Json))
}
catch {
throw "checkpoint timeline line $lineNumber is invalid JSON: $($_.Exception.Message)"
}
}
return @($records)
}
function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
if ($canonicalCheckpoints.Count -ne $expectedCheckpointNames.Count) {
$failures.Add("expected $($expectedCheckpointNames.Count) canonical checkpoints, found $($canonicalCheckpoints.Count)")
}
$count = [Math]::Min($canonicalCheckpoints.Count, $expectedCheckpointNames.Count)
for ($index = 0; $index -lt $count; $index++) {
$checkpoint = $canonicalCheckpoints[$index]
$expectedName = $expectedCheckpointNames[$index]
$expectedSequence = $index + 1
if ($checkpoint.name -ne $expectedName) {
$failures.Add("canonical checkpoint $expectedSequence was '$($checkpoint.name)', expected '$expectedName'")
}
if ([int]$checkpoint.sequence -ne $expectedSequence) {
$failures.Add("canonical checkpoint '$expectedName' had sequence $($checkpoint.sequence), expected $expectedSequence")
}
if ([int]$checkpoint.processId -ne $Client.Id) {
$failures.Add("canonical checkpoint '$expectedName' came from process $($checkpoint.processId), expected $($Client.Id)")
}
if ($null -eq $checkpoint.render -or $null -eq $checkpoint.resources) {
$failures.Add("canonical checkpoint '$expectedName' is missing render/resources data")
continue
}
$renderWorldProperty = $checkpoint.render.PSObject.Properties['world']
$renderVisibleProperty = if ($null -ne $renderWorldProperty) {
$renderWorldProperty.Value.PSObject.Properties['visibleLandblocks']
}
$renderTotalProperty = if ($null -ne $renderWorldProperty) {
$renderWorldProperty.Value.PSObject.Properties['totalLandblocks']
}
$resourceVisibleProperty =
$checkpoint.resources.PSObject.Properties['visibleLandblocks']
$resourceTotalProperty =
$checkpoint.resources.PSObject.Properties['totalLandblocks']
if ($null -eq $renderWorldProperty -or
$null -eq $renderVisibleProperty -or
$null -eq $renderTotalProperty -or
$null -eq $resourceVisibleProperty -or
$null -eq $resourceTotalProperty) {
$failures.Add("canonical checkpoint '$expectedName' is missing render-world synchronization fields")
}
elseif ([int]$renderVisibleProperty.Value -ne
[int]$resourceVisibleProperty.Value) {
$failures.Add("canonical checkpoint '$expectedName' mixed render/resource visible-landblock frames")
}
elseif ([int]$renderTotalProperty.Value -ne
[int]$resourceTotalProperty.Value) {
$failures.Add("canonical checkpoint '$expectedName' mixed render/resource total-landblock frames")
}
$namedCheckpointPath =
Join-Path $artifactDir "checkpoint-$expectedName.json"
if (-not (Test-Path -LiteralPath $namedCheckpointPath)) {
$failures.Add("canonical checkpoint '$expectedName' is missing named artifact '$namedCheckpointPath'")
}
else {
try {
$namedCheckpoint =
Get-Content -LiteralPath $namedCheckpointPath -Raw |
ConvertFrom-Json
if ($namedCheckpoint.name -ne $checkpoint.name -or
[int]$namedCheckpoint.sequence -ne [int]$checkpoint.sequence -or
[int]$namedCheckpoint.processId -ne [int]$checkpoint.processId) {
$failures.Add("canonical checkpoint '$expectedName' named artifact does not match its timeline row")
}
}
catch {
$failures.Add("canonical checkpoint '$expectedName' named artifact is invalid JSON: $($_.Exception.Message)")
}
}
foreach ($field in @(
'pendingLiveTeardowns',
'pendingLandblockRetirements',
'stagedMeshUploads',
'stagedMeshBytes',
'compositeWarmupPending')) {
$property = $checkpoint.resources.PSObject.Properties[$field]
if ($null -eq $property) {
$failures.Add("canonical checkpoint '$expectedName' is missing resources.$field")
}
elseif ([long]$property.Value -ne 0) {
$failures.Add("canonical checkpoint '$expectedName' has resources.$field=$($property.Value), expected zero")
}
}
}
$caulReturn = $canonicalCheckpoints |
Where-Object { $_.name -eq 'caul-return' } |
Select-Object -First 1
$caulPlateau = $canonicalCheckpoints |
Where-Object { $_.name -eq 'caul-plateau' } |
Select-Object -First 1
if ($null -eq $caulReturn -or $null -eq $caulPlateau) {
$failures.Add('canonical Caul return/plateau comparison is unavailable')
return
}
foreach ($field in @(
'loadedLandblocks',
'visibleLandblocks',
'totalLandblocks',
'pendingLiveTeardowns',
'pendingLandblockRetirements',
'meshRenderData',
'meshAtlasArrays',
'meshEstimatedBytes',
'stagedMeshUploads',
'stagedMeshBytes',
'trackedGpuBytes',
'trackedGpuBuffers',
'trackedGpuTextures',
'ownedCompositeTextures',
'compositeTextureOwners',
'activeParticleTextures',
'particleTextureOwners',
'compositeWarmupPending')) {
$first = $caulReturn.resources.PSObject.Properties[$field]
$plateau = $caulPlateau.resources.PSObject.Properties[$field]
if ($null -eq $first -or $null -eq $plateau) {
$failures.Add("canonical Caul comparison is missing resources.$field")
}
elseif ([long]$first.Value -ne [long]$plateau.Value) {
$failures.Add("Caul plateau canonical owner '$field' changed $($first.Value) -> $($plateau.Value)")
}
}
$populationChanged = $false
foreach ($field in @(
'worldEntities',
'animatedEntities',
'liveEntities',
'materializedLiveEntities')) {
$first = $caulReturn.resources.PSObject.Properties[$field]
$plateau = $caulPlateau.resources.PSObject.Properties[$field]
if ($null -eq $first -or $null -eq $plateau) {
$failures.Add("canonical Caul population comparison is missing resources.$field")
}
elseif ([long]$first.Value -ne [long]$plateau.Value) {
$populationChanged = $true
$warnings.Add("Caul plateau authoritative population '$field' changed $($first.Value) -> $($plateau.Value)")
}
}
foreach ($field in @(
'particleBindings',
'particleOwners',
'effectOwners',
'lightOwners',
'scriptOwners')) {
$first = $caulReturn.resources.PSObject.Properties[$field]
$plateau = $caulPlateau.resources.PSObject.Properties[$field]
if ($null -eq $first -or $null -eq $plateau) {
$failures.Add("canonical Caul owner comparison is missing resources.$field")
}
elseif ([long]$first.Value -ne [long]$plateau.Value) {
$message = "Caul plateau population-sensitive owner '$field' changed $($first.Value) -> $($plateau.Value)"
if ($populationChanged) { $warnings.Add($message) }
else { $failures.Add($message) }
}
}
foreach ($field in @(
'particleEmitters',
'particles',
'activeScripts')) {
$first = $caulReturn.resources.PSObject.Properties[$field]
$plateau = $caulPlateau.resources.PSObject.Properties[$field]
if ($null -eq $first -or $null -eq $plateau) {
$failures.Add("canonical Caul transient comparison is missing resources.$field")
}
elseif ([long]$first.Value -ne [long]$plateau.Value) {
$warnings.Add("Caul plateau authoritative/transient workload '$field' changed $($first.Value) -> $($plateau.Value)")
}
}
}
function Add-RelativeGateFailures { function Add-RelativeGateFailures {
$caulReturn = $samples | Where-Object { $_.Destination -eq 'Caul return' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 $caulReturn = $samples | Where-Object { $_.Destination -eq 'Caul return' -and $_.Phase -eq 'stationary' } | Select-Object -First 1
$caulPlateau = $samples | Where-Object { $_.Destination -eq 'Caul plateau' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 $caulPlateau = $samples | Where-Object { $_.Destination -eq 'Caul plateau' -and $_.Phase -eq 'stationary' } | Select-Object -First 1
@ -430,6 +640,14 @@ try {
$failures.Add("expected $($destinations.Count) teleport materializations") $failures.Add("expected $($destinations.Count) teleport materializations")
} }
$null = Wait-ForLogPattern `
$process `
$stdoutLog `
'[world-gate] checkpoint name=caul-plateau sequence=9' `
1 `
30
$canonicalCheckpoints = @(Read-CanonicalCheckpoints)
Add-CanonicalCheckpointFailures $process
Add-RelativeGateFailures Add-RelativeGateFailures
Write-Marker 'route-complete requesting graceful close' Write-Marker 'route-complete requesting graceful close'
@ -474,12 +692,14 @@ finally {
SourceStatus = $sourceStatus SourceStatus = $sourceStatus
VideoControllers = $videoControllers VideoControllers = $videoControllers
Samples = @($samples) Samples = @($samples)
CanonicalCheckpoints = @($canonicalCheckpoints)
Artifacts = [pscustomobject]@{ Artifacts = [pscustomobject]@{
Stdout = $stdoutLog Stdout = $stdoutLog
Stderr = $stderrLog Stderr = $stderrLog
Markers = $markerLog Markers = $markerLog
SamplesCsv = $sampleCsv SamplesCsv = $sampleCsv
ReportJson = $reportJson ReportJson = $reportJson
CheckpointTimeline = $checkpointTimeline
} }
} }
$report | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $reportJson -Encoding utf8 $report | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $reportJson -Encoding utf8