From 4a205a3e56ebf89850198b6d1994c380ea48ed1a Mon Sep 17 00:00:00 2001 From: Erik Date: Tue, 21 Jul 2026 23:41:15 +0200 Subject: [PATCH] test(streaming): close lifecycle gate after cutover --- src/AcDream.App/Net/LiveSessionController.cs | 5 ++ src/AcDream.App/Rendering/GameWindow.cs | 30 ++++++++--- .../Net/LiveSessionControllerTests.cs | 43 +++++++++++++++ .../Streaming/LandblockBuildOriginTests.cs | 52 +++++++++++++++++++ tools/connected-r6-soak.route.txt | 27 ++++++++++ tools/run-connected-r6-soak.ps1 | 31 +++++++---- tools/run-connected-world-lifecycle-gate.ps1 | 2 + 7 files changed, 173 insertions(+), 17 deletions(-) diff --git a/src/AcDream.App/Net/LiveSessionController.cs b/src/AcDream.App/Net/LiveSessionController.cs index 13acfa40..eecab525 100644 --- a/src/AcDream.App/Net/LiveSessionController.cs +++ b/src/AcDream.App/Net/LiveSessionController.cs @@ -259,6 +259,11 @@ public sealed class LiveSessionController : IDisposable get { lock (_gate) return _generation; } } + internal bool IsDisposalComplete + { + get { lock (_gate) return _disposed; } + } + internal LiveSessionStartResult Start( RuntimeOptions options, ILiveSessionLifecycleHost host) diff --git a/src/AcDream.App/Rendering/GameWindow.cs b/src/AcDream.App/Rendering/GameWindow.cs index e6a816be..e4a0f07e 100644 --- a/src/AcDream.App/Rendering/GameWindow.cs +++ b/src/AcDream.App/Rendering/GameWindow.cs @@ -8606,7 +8606,30 @@ public sealed class GameWindow : IDisposable, ILiveAnimationPresentationContext } private ResourceShutdownTransaction CreateShutdownTransaction() => new( - new ResourceShutdownStage("session owners", + // Live-session reset retires the complete streaming window. Every + // reset callback owner, especially LandblockStreamer, must remain live + // until that transaction converges. + new ResourceShutdownStage("session lifetime", + [ + new("live session", () => + { + AcDream.App.Net.LiveSessionController? controller = + _liveSessionController; + if (controller is null) + return; + + controller.Dispose(); + if (!controller.IsDisposalComplete) + { + throw new InvalidOperationException( + "Live-session disposal was deferred by a reentrant lifecycle callback."); + } + + if (ReferenceEquals(_liveSessionController, controller)) + _liveSessionController = null; + }), + ]), + new ResourceShutdownStage("session dependents", [ new("mouse capture", EndMouseLookAndRestoreCursor), new("retail UI", () => @@ -8641,11 +8664,6 @@ public sealed class GameWindow : IDisposable, ILiveAnimationPresentationContext _magicCatalog = null; }), new("streamer", () => _streamer?.Dispose()), - new("live session", () => - { - _liveSessionController?.Dispose(); - _liveSessionController = null; - }), new("equipped children", () => _equippedChildRenderer?.Dispose()), ]), // Retained tombstones are retried while every callback owner is alive. diff --git a/tests/AcDream.App.Tests/Net/LiveSessionControllerTests.cs b/tests/AcDream.App.Tests/Net/LiveSessionControllerTests.cs index 3c56847d..4b4539fc 100644 --- a/tests/AcDream.App.Tests/Net/LiveSessionControllerTests.cs +++ b/tests/AcDream.App.Tests/Net/LiveSessionControllerTests.cs @@ -1,5 +1,6 @@ using System.Net; using AcDream.App.Net; +using AcDream.App.Rendering; using AcDream.Core.Net; using AcDream.Core.Net.Messages; using AcDream.UI.Abstractions; @@ -833,6 +834,48 @@ public sealed class LiveSessionControllerTests Assert.Throws(() => controller.Start(LiveOptions(), host)); } + [Fact] + public void ReentrantShutdownRetainsControllerAndDependentsUntilDeferredDisposeCompletes() + { + var calls = new List(); + var operations = new TestOperations(calls); + var host = new TestHost(calls); + var controller = new LiveSessionController(operations); + controller.Start(LiveOptions(), host); + bool dependentDisposed = false; + var shutdown = new ResourceShutdownTransaction( + new ResourceShutdownStage("session lifetime", + [ + new("live session", () => + { + controller.Dispose(); + if (!controller.IsDisposalComplete) + { + throw new InvalidOperationException( + "live-session disposal is still deferred"); + } + }), + ]), + new ResourceShutdownStage("session dependents", + [ + new("streamer", () => dependentDisposed = true), + ])); + operations.OnTick = () => + { + Assert.Throws(shutdown.CompleteOrThrow); + Assert.Equal(0, shutdown.CurrentStage); + Assert.False(dependentDisposed); + }; + + controller.Tick(); + + Assert.True(controller.IsDisposalComplete); + Assert.False(dependentDisposed); + shutdown.CompleteOrThrow(); + Assert.True(shutdown.IsComplete); + Assert.True(dependentDisposed); + } + private static RuntimeOptions LiveOptions( bool live = true, string? user = "user") diff --git a/tests/AcDream.App.Tests/Streaming/LandblockBuildOriginTests.cs b/tests/AcDream.App.Tests/Streaming/LandblockBuildOriginTests.cs index 9553ea8a..db902231 100644 --- a/tests/AcDream.App.Tests/Streaming/LandblockBuildOriginTests.cs +++ b/tests/AcDream.App.Tests/Streaming/LandblockBuildOriginTests.cs @@ -342,6 +342,58 @@ public sealed class LandblockBuildOriginTests Assert.DoesNotContain("_liveCenterY", renderPublisherSource, StringComparison.Ordinal); } + [Fact] + public void GameWindowShutdownKeepsStreamerAliveUntilSessionResetConverges() + { + string source = File.ReadAllText(Path.Combine( + FindRepoRoot(), + "src", + "AcDream.App", + "Rendering", + "GameWindow.cs")); + int sessionStage = source.IndexOf( + "new ResourceShutdownStage(\"session lifetime\"", + StringComparison.Ordinal); + Assert.True(sessionStage >= 0); + + int sessionOperation = source.IndexOf( + "new(\"live session\", () =>", + sessionStage, + StringComparison.Ordinal); + Assert.True(sessionOperation > sessionStage); + + int sessionDispose = source.IndexOf( + "controller.Dispose()", + sessionOperation, + StringComparison.Ordinal); + Assert.True(sessionDispose > sessionOperation); + + int disposalCompletionBarrier = source.IndexOf( + "if (!controller.IsDisposalComplete)", + sessionDispose, + StringComparison.Ordinal); + Assert.True(disposalCompletionBarrier > sessionDispose); + + int sessionReferenceRelease = source.IndexOf( + "_liveSessionController = null;", + disposalCompletionBarrier, + StringComparison.Ordinal); + Assert.True(sessionReferenceRelease > disposalCompletionBarrier); + + int dependentStage = source.IndexOf( + "new ResourceShutdownStage(\"session dependents\"", + sessionReferenceRelease, + StringComparison.Ordinal); + Assert.True(dependentStage > sessionReferenceRelease); + + int streamerDispose = source.IndexOf( + "new(\"streamer\", () => _streamer?.Dispose())", + dependentStage, + StringComparison.Ordinal); + + Assert.True(streamerDispose > dependentStage); + } + private static LandblockBuild EmptyBuild(uint landblockId, LandblockBuildOrigin origin) => new( new LoadedLandblock( diff --git a/tools/connected-r6-soak.route.txt b/tools/connected-r6-soak.route.txt index 4e7b457b..a99d417b 100644 --- a/tools/connected-r6-soak.route.txt +++ b/tools/connected-r6-soak.route.txt @@ -6,6 +6,7 @@ sleep 15000 # Caul baseline + local movement/jump/combat exercise. command /teleloc 0xC95B0001 14.8 0.3 12.005 +wait materialized 1 60000 sleep 8000 input down MovementTurnRight sleep 6000 @@ -42,6 +43,7 @@ sleep 7400 # Sawato baseline. command /teleloc 0x3032001C 83.3 89.2 133.005 +wait materialized 2 60000 sleep 8000 input down MovementTurnRight sleep 6000 @@ -50,6 +52,7 @@ sleep 22000 # Rynthid world-edge streaming. command /teleloc 0xF6820033 145.7 49.855 58.010 +wait materialized 3 60000 sleep 8000 input down MovementTurnRight sleep 6000 @@ -58,6 +61,7 @@ sleep 22000 # Aerlinthe dense island. command /teleloc 0x09040008 11.4 188.6 87.705 +wait materialized 4 60000 sleep 8000 input down MovementTurnRight sleep 6000 @@ -66,6 +70,7 @@ sleep 22000 # Same-location lifecycle revisit. command /teleloc 0x3032001C 83.3 89.2 133.005 +wait materialized 5 60000 sleep 8000 input down MovementTurnRight sleep 6000 @@ -74,6 +79,7 @@ sleep 22000 # Holtburg mixed-town workload + local exercise. command /teleloc 0xA9B40019 84.0 7.1 94.005 +wait materialized 6 60000 sleep 8000 input down MovementTurnRight sleep 6000 @@ -110,6 +116,7 @@ sleep 7400 # Caul return/leak oracle + final local exercise. command /teleloc 0xC95B0001 14.8 0.3 12.005 +wait materialized 7 60000 sleep 8000 input down MovementTurnRight sleep 6000 @@ -143,3 +150,23 @@ input up MovementForward sleep 700 input press CombatToggleCombat sleep 15000 + +# A second warm-cache Sawato/Caul revisit is the plateau oracle. The first +# circuit intentionally warms bounded DAT/GPU caches, so comparing the cold +# first Caul against its warm return would punish successful source-window +# retirement. This circuit loads no new destination content. +command /teleloc 0x3032001C 83.3 89.2 133.005 +wait materialized 8 60000 +sleep 8000 +input down MovementTurnRight +sleep 6000 +input up MovementTurnRight +sleep 22000 + +command /teleloc 0xC95B0001 14.8 0.3 12.005 +wait materialized 9 60000 +sleep 8000 +input down MovementTurnRight +sleep 6000 +input up MovementTurnRight +sleep 22000 diff --git a/tools/run-connected-r6-soak.ps1 b/tools/run-connected-r6-soak.ps1 index 1afbaf6b..460d0c56 100644 --- a/tools/run-connected-r6-soak.ps1 +++ b/tools/run-connected-r6-soak.ps1 @@ -20,7 +20,9 @@ $destinations = @( [pscustomobject]@{ Name = 'Aerlinthe'; Occurrence = 4; Exercise = $false }, [pscustomobject]@{ Name = 'Sawato revisit'; Occurrence = 5; Exercise = $false }, [pscustomobject]@{ Name = 'Holtburg'; Occurrence = 6; Exercise = $true }, - [pscustomobject]@{ Name = 'Caul return'; Occurrence = 7; Exercise = $true } + [pscustomobject]@{ Name = 'Caul return'; Occurrence = 7; Exercise = $true }, + [pscustomobject]@{ Name = 'Sawato plateau'; Occurrence = 8; Exercise = $false }, + [pscustomobject]@{ Name = 'Caul plateau'; Occurrence = 9; Exercise = $false } ) $stamp = Get-Date -Format 'yyyyMMdd-HHmmss' @@ -32,6 +34,7 @@ $stderrLog = "$prefix.err.log" $markerLog = "$prefix.markers.log" $sampleCsv = "$prefix.samples.csv" $reportJson = "$prefix.report.json" +$artifactDir = "$prefix.artifacts" $routePath = Join-Path $Repository 'tools\connected-r6-soak.route.txt' $exe = Join-Path $Repository 'src\AcDream.App\bin\Release\net10.0\AcDream.App.exe' @@ -59,7 +62,7 @@ function Get-PatternCount([string]$Path, [string]$Pattern) { } function Get-FrameProfiles([string]$Path) { - return @(Read-LogLines $Path | Where-Object { $_.StartsWith('[frame-prof]', [StringComparison]::Ordinal) }) + return @(Read-LogLines $Path | Where-Object { $_.StartsWith('[frame-prof] n=', [StringComparison]::Ordinal) }) } function Wait-ForLogPattern( @@ -206,6 +209,12 @@ function Capture-Sample( Profile = $ProfileLine } $samples.Add($sample) + if ($null -eq $sample.ProfileFrames -or + $null -eq $sample.CpuP95Ms -or + $null -eq $sample.AllocP50Kb -or + $null -eq $sample.UpdateP95Ms) { + $failures.Add("${Destination}/${Phase}: frame-prof sample was missing required parsed metrics") + } return $sample } @@ -245,14 +254,11 @@ function Observe-R6Exercise( } function Add-RelativeGateFailures { - $caulFirst = $samples | Where-Object { $_.Destination -eq 'Caul' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 $caulReturn = $samples | Where-Object { $_.Destination -eq 'Caul return' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 - $sawatoFirst = $samples | Where-Object { $_.Destination -eq 'Sawato' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 - $sawatoReturn = $samples | Where-Object { $_.Destination -eq 'Sawato revisit' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 + $caulPlateau = $samples | Where-Object { $_.Destination -eq 'Caul plateau' -and $_.Phase -eq 'stationary' } | Select-Object -First 1 foreach ($pair in @( - [pscustomobject]@{ Label = 'Caul return'; First = $caulFirst; Return = $caulReturn }, - [pscustomobject]@{ Label = 'Sawato revisit'; First = $sawatoFirst; Return = $sawatoReturn })) + [pscustomobject]@{ Label = 'Caul plateau'; First = $caulReturn; Return = $caulPlateau })) { if ($null -eq $pair.First -or $null -eq $pair.Return) { $failures.Add("$($pair.Label): missing same-location stationary sample") @@ -273,8 +279,8 @@ function Add-RelativeGateFailures { $warnings.Add("$($pair.Label): connected animation population changed $($pair.First.AnimationCount) -> $($pair.Return.AnimationCount)") } - $privateLimit = [Math]::Max(384.0, $pair.First.PrivateMiB * 0.35) - $workingLimit = [Math]::Max(384.0, $pair.First.WorkingSetMiB * 0.35) + $privateLimit = [Math]::Max(192.0, $pair.First.PrivateMiB * 0.20) + $workingLimit = [Math]::Max(192.0, $pair.First.WorkingSetMiB * 0.20) if (($pair.Return.PrivateMiB - $pair.First.PrivateMiB) -gt $privateLimit) { $failures.Add("$($pair.Label): private memory grew by $([Math]::Round($pair.Return.PrivateMiB - $pair.First.PrivateMiB, 1)) MiB") } @@ -305,7 +311,9 @@ function Add-LogFailures { 'WeenieError', 'device removed', 'GPU reset', - 'live: disconnected' + 'live: disconnected', + '[shutdown]', + 'ObjectDisposedException' ) foreach ($pattern in $fatalPatterns) { $count = (Get-PatternCount $stdoutLog $pattern) + (Get-PatternCount $stderrLog $pattern) @@ -369,6 +377,7 @@ $env:ACDREAM_UNCAPPED_RENDER = '1' $env:ACDREAM_DEVTOOLS = '0' $env:ACDREAM_UI_PROBE_DUMP = '0' $env:ACDREAM_UI_PROBE_SCRIPT = $routePath +$env:ACDREAM_AUTOMATION_ARTIFACT_DIR = $artifactDir $env:ACDREAM_DUMP_MOVE_TRUTH = '1' $env:ACDREAM_NO_AUDIO = $null $env:ACDREAM_WB_DIAG = $null @@ -422,7 +431,6 @@ try { } Add-RelativeGateFailures - Add-LogFailures Write-Marker 'route-complete requesting graceful close' $gracefulExit = Close-ClientGracefully $process @@ -430,6 +438,7 @@ try { if ($process.HasExited) { $exitCode = [int]$process.ExitCode } if (-not $gracefulExit) { $failures.Add('client did not exit through the graceful WM_CLOSE path') } if ($null -ne $exitCode -and $exitCode -ne 0) { $failures.Add("client exit code was $exitCode") } + Add-LogFailures } catch { $failures.Add($_.Exception.Message) diff --git a/tools/run-connected-world-lifecycle-gate.ps1 b/tools/run-connected-world-lifecycle-gate.ps1 index f0c599ef..d03efc08 100644 --- a/tools/run-connected-world-lifecycle-gate.ps1 +++ b/tools/run-connected-world-lifecycle-gate.ps1 @@ -113,6 +113,8 @@ function Add-LogFailures([string]$Label, [string]$Stdout, [string]$Stderr) { 'device removed', 'GPU reset', 'live: disconnected', + '[shutdown]', + 'ObjectDisposedException', 'screenshot-failed', 'graceful logout confirmation timed out', 'graceful logout failed',