test(streaming): close lifecycle gate after cutover
This commit is contained in:
parent
db3ca59fd0
commit
4a205a3e56
7 changed files with 173 additions and 17 deletions
|
|
@ -259,6 +259,11 @@ public sealed class LiveSessionController : IDisposable
|
|||
get { lock (_gate) return _generation; }
|
||||
}
|
||||
|
||||
internal bool IsDisposalComplete
|
||||
{
|
||||
get { lock (_gate) return _disposed; }
|
||||
}
|
||||
|
||||
internal LiveSessionStartResult Start(
|
||||
RuntimeOptions options,
|
||||
ILiveSessionLifecycleHost host)
|
||||
|
|
|
|||
|
|
@ -8606,7 +8606,30 @@ public sealed class GameWindow : IDisposable, ILiveAnimationPresentationContext
|
|||
}
|
||||
|
||||
private ResourceShutdownTransaction CreateShutdownTransaction() => new(
|
||||
new ResourceShutdownStage("session owners",
|
||||
// Live-session reset retires the complete streaming window. Every
|
||||
// reset callback owner, especially LandblockStreamer, must remain live
|
||||
// until that transaction converges.
|
||||
new ResourceShutdownStage("session lifetime",
|
||||
[
|
||||
new("live session", () =>
|
||||
{
|
||||
AcDream.App.Net.LiveSessionController? controller =
|
||||
_liveSessionController;
|
||||
if (controller is null)
|
||||
return;
|
||||
|
||||
controller.Dispose();
|
||||
if (!controller.IsDisposalComplete)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Live-session disposal was deferred by a reentrant lifecycle callback.");
|
||||
}
|
||||
|
||||
if (ReferenceEquals(_liveSessionController, controller))
|
||||
_liveSessionController = null;
|
||||
}),
|
||||
]),
|
||||
new ResourceShutdownStage("session dependents",
|
||||
[
|
||||
new("mouse capture", EndMouseLookAndRestoreCursor),
|
||||
new("retail UI", () =>
|
||||
|
|
@ -8641,11 +8664,6 @@ public sealed class GameWindow : IDisposable, ILiveAnimationPresentationContext
|
|||
_magicCatalog = null;
|
||||
}),
|
||||
new("streamer", () => _streamer?.Dispose()),
|
||||
new("live session", () =>
|
||||
{
|
||||
_liveSessionController?.Dispose();
|
||||
_liveSessionController = null;
|
||||
}),
|
||||
new("equipped children", () => _equippedChildRenderer?.Dispose()),
|
||||
]),
|
||||
// Retained tombstones are retried while every callback owner is alive.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
using System.Net;
|
||||
using AcDream.App.Net;
|
||||
using AcDream.App.Rendering;
|
||||
using AcDream.Core.Net;
|
||||
using AcDream.Core.Net.Messages;
|
||||
using AcDream.UI.Abstractions;
|
||||
|
|
@ -833,6 +834,48 @@ public sealed class LiveSessionControllerTests
|
|||
Assert.Throws<ObjectDisposedException>(() => controller.Start(LiveOptions(), host));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ReentrantShutdownRetainsControllerAndDependentsUntilDeferredDisposeCompletes()
|
||||
{
|
||||
var calls = new List<string>();
|
||||
var operations = new TestOperations(calls);
|
||||
var host = new TestHost(calls);
|
||||
var controller = new LiveSessionController(operations);
|
||||
controller.Start(LiveOptions(), host);
|
||||
bool dependentDisposed = false;
|
||||
var shutdown = new ResourceShutdownTransaction(
|
||||
new ResourceShutdownStage("session lifetime",
|
||||
[
|
||||
new("live session", () =>
|
||||
{
|
||||
controller.Dispose();
|
||||
if (!controller.IsDisposalComplete)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"live-session disposal is still deferred");
|
||||
}
|
||||
}),
|
||||
]),
|
||||
new ResourceShutdownStage("session dependents",
|
||||
[
|
||||
new("streamer", () => dependentDisposed = true),
|
||||
]));
|
||||
operations.OnTick = () =>
|
||||
{
|
||||
Assert.Throws<AggregateException>(shutdown.CompleteOrThrow);
|
||||
Assert.Equal(0, shutdown.CurrentStage);
|
||||
Assert.False(dependentDisposed);
|
||||
};
|
||||
|
||||
controller.Tick();
|
||||
|
||||
Assert.True(controller.IsDisposalComplete);
|
||||
Assert.False(dependentDisposed);
|
||||
shutdown.CompleteOrThrow();
|
||||
Assert.True(shutdown.IsComplete);
|
||||
Assert.True(dependentDisposed);
|
||||
}
|
||||
|
||||
private static RuntimeOptions LiveOptions(
|
||||
bool live = true,
|
||||
string? user = "user")
|
||||
|
|
|
|||
|
|
@ -342,6 +342,58 @@ public sealed class LandblockBuildOriginTests
|
|||
Assert.DoesNotContain("_liveCenterY", renderPublisherSource, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void GameWindowShutdownKeepsStreamerAliveUntilSessionResetConverges()
|
||||
{
|
||||
string source = File.ReadAllText(Path.Combine(
|
||||
FindRepoRoot(),
|
||||
"src",
|
||||
"AcDream.App",
|
||||
"Rendering",
|
||||
"GameWindow.cs"));
|
||||
int sessionStage = source.IndexOf(
|
||||
"new ResourceShutdownStage(\"session lifetime\"",
|
||||
StringComparison.Ordinal);
|
||||
Assert.True(sessionStage >= 0);
|
||||
|
||||
int sessionOperation = source.IndexOf(
|
||||
"new(\"live session\", () =>",
|
||||
sessionStage,
|
||||
StringComparison.Ordinal);
|
||||
Assert.True(sessionOperation > sessionStage);
|
||||
|
||||
int sessionDispose = source.IndexOf(
|
||||
"controller.Dispose()",
|
||||
sessionOperation,
|
||||
StringComparison.Ordinal);
|
||||
Assert.True(sessionDispose > sessionOperation);
|
||||
|
||||
int disposalCompletionBarrier = source.IndexOf(
|
||||
"if (!controller.IsDisposalComplete)",
|
||||
sessionDispose,
|
||||
StringComparison.Ordinal);
|
||||
Assert.True(disposalCompletionBarrier > sessionDispose);
|
||||
|
||||
int sessionReferenceRelease = source.IndexOf(
|
||||
"_liveSessionController = null;",
|
||||
disposalCompletionBarrier,
|
||||
StringComparison.Ordinal);
|
||||
Assert.True(sessionReferenceRelease > disposalCompletionBarrier);
|
||||
|
||||
int dependentStage = source.IndexOf(
|
||||
"new ResourceShutdownStage(\"session dependents\"",
|
||||
sessionReferenceRelease,
|
||||
StringComparison.Ordinal);
|
||||
Assert.True(dependentStage > sessionReferenceRelease);
|
||||
|
||||
int streamerDispose = source.IndexOf(
|
||||
"new(\"streamer\", () => _streamer?.Dispose())",
|
||||
dependentStage,
|
||||
StringComparison.Ordinal);
|
||||
|
||||
Assert.True(streamerDispose > dependentStage);
|
||||
}
|
||||
|
||||
private static LandblockBuild EmptyBuild(uint landblockId, LandblockBuildOrigin origin) =>
|
||||
new(
|
||||
new LoadedLandblock(
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ sleep 15000
|
|||
|
||||
# Caul baseline + local movement/jump/combat exercise.
|
||||
command /teleloc 0xC95B0001 14.8 0.3 12.005
|
||||
wait materialized 1 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
|
|
@ -42,6 +43,7 @@ sleep 7400
|
|||
|
||||
# Sawato baseline.
|
||||
command /teleloc 0x3032001C 83.3 89.2 133.005
|
||||
wait materialized 2 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
|
|
@ -50,6 +52,7 @@ sleep 22000
|
|||
|
||||
# Rynthid world-edge streaming.
|
||||
command /teleloc 0xF6820033 145.7 49.855 58.010
|
||||
wait materialized 3 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
|
|
@ -58,6 +61,7 @@ sleep 22000
|
|||
|
||||
# Aerlinthe dense island.
|
||||
command /teleloc 0x09040008 11.4 188.6 87.705
|
||||
wait materialized 4 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
|
|
@ -66,6 +70,7 @@ sleep 22000
|
|||
|
||||
# Same-location lifecycle revisit.
|
||||
command /teleloc 0x3032001C 83.3 89.2 133.005
|
||||
wait materialized 5 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
|
|
@ -74,6 +79,7 @@ sleep 22000
|
|||
|
||||
# Holtburg mixed-town workload + local exercise.
|
||||
command /teleloc 0xA9B40019 84.0 7.1 94.005
|
||||
wait materialized 6 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
|
|
@ -110,6 +116,7 @@ sleep 7400
|
|||
|
||||
# Caul return/leak oracle + final local exercise.
|
||||
command /teleloc 0xC95B0001 14.8 0.3 12.005
|
||||
wait materialized 7 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
|
|
@ -143,3 +150,23 @@ input up MovementForward
|
|||
sleep 700
|
||||
input press CombatToggleCombat
|
||||
sleep 15000
|
||||
|
||||
# A second warm-cache Sawato/Caul revisit is the plateau oracle. The first
|
||||
# circuit intentionally warms bounded DAT/GPU caches, so comparing the cold
|
||||
# first Caul against its warm return would punish successful source-window
|
||||
# retirement. This circuit loads no new destination content.
|
||||
command /teleloc 0x3032001C 83.3 89.2 133.005
|
||||
wait materialized 8 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
input up MovementTurnRight
|
||||
sleep 22000
|
||||
|
||||
command /teleloc 0xC95B0001 14.8 0.3 12.005
|
||||
wait materialized 9 60000
|
||||
sleep 8000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
input up MovementTurnRight
|
||||
sleep 22000
|
||||
|
|
|
|||
|
|
@ -20,7 +20,9 @@ $destinations = @(
|
|||
[pscustomobject]@{ Name = 'Aerlinthe'; Occurrence = 4; Exercise = $false },
|
||||
[pscustomobject]@{ Name = 'Sawato revisit'; Occurrence = 5; Exercise = $false },
|
||||
[pscustomobject]@{ Name = 'Holtburg'; Occurrence = 6; Exercise = $true },
|
||||
[pscustomobject]@{ Name = 'Caul return'; Occurrence = 7; Exercise = $true }
|
||||
[pscustomobject]@{ Name = 'Caul return'; Occurrence = 7; Exercise = $true },
|
||||
[pscustomobject]@{ Name = 'Sawato plateau'; Occurrence = 8; Exercise = $false },
|
||||
[pscustomobject]@{ Name = 'Caul plateau'; Occurrence = 9; Exercise = $false }
|
||||
)
|
||||
|
||||
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
|
|
@ -32,6 +34,7 @@ $stderrLog = "$prefix.err.log"
|
|||
$markerLog = "$prefix.markers.log"
|
||||
$sampleCsv = "$prefix.samples.csv"
|
||||
$reportJson = "$prefix.report.json"
|
||||
$artifactDir = "$prefix.artifacts"
|
||||
$routePath = Join-Path $Repository 'tools\connected-r6-soak.route.txt'
|
||||
$exe = Join-Path $Repository 'src\AcDream.App\bin\Release\net10.0\AcDream.App.exe'
|
||||
|
||||
|
|
@ -59,7 +62,7 @@ function Get-PatternCount([string]$Path, [string]$Pattern) {
|
|||
}
|
||||
|
||||
function Get-FrameProfiles([string]$Path) {
|
||||
return @(Read-LogLines $Path | Where-Object { $_.StartsWith('[frame-prof]', [StringComparison]::Ordinal) })
|
||||
return @(Read-LogLines $Path | Where-Object { $_.StartsWith('[frame-prof] n=', [StringComparison]::Ordinal) })
|
||||
}
|
||||
|
||||
function Wait-ForLogPattern(
|
||||
|
|
@ -206,6 +209,12 @@ function Capture-Sample(
|
|||
Profile = $ProfileLine
|
||||
}
|
||||
$samples.Add($sample)
|
||||
if ($null -eq $sample.ProfileFrames -or
|
||||
$null -eq $sample.CpuP95Ms -or
|
||||
$null -eq $sample.AllocP50Kb -or
|
||||
$null -eq $sample.UpdateP95Ms) {
|
||||
$failures.Add("${Destination}/${Phase}: frame-prof sample was missing required parsed metrics")
|
||||
}
|
||||
return $sample
|
||||
}
|
||||
|
||||
|
|
@ -245,14 +254,11 @@ function Observe-R6Exercise(
|
|||
}
|
||||
|
||||
function Add-RelativeGateFailures {
|
||||
$caulFirst = $samples | Where-Object { $_.Destination -eq 'Caul' -and $_.Phase -eq 'stationary' } | Select-Object -First 1
|
||||
$caulReturn = $samples | Where-Object { $_.Destination -eq 'Caul return' -and $_.Phase -eq 'stationary' } | Select-Object -First 1
|
||||
$sawatoFirst = $samples | Where-Object { $_.Destination -eq 'Sawato' -and $_.Phase -eq 'stationary' } | Select-Object -First 1
|
||||
$sawatoReturn = $samples | Where-Object { $_.Destination -eq 'Sawato revisit' -and $_.Phase -eq 'stationary' } | Select-Object -First 1
|
||||
$caulPlateau = $samples | Where-Object { $_.Destination -eq 'Caul plateau' -and $_.Phase -eq 'stationary' } | Select-Object -First 1
|
||||
|
||||
foreach ($pair in @(
|
||||
[pscustomobject]@{ Label = 'Caul return'; First = $caulFirst; Return = $caulReturn },
|
||||
[pscustomobject]@{ Label = 'Sawato revisit'; First = $sawatoFirst; Return = $sawatoReturn }))
|
||||
[pscustomobject]@{ Label = 'Caul plateau'; First = $caulReturn; Return = $caulPlateau }))
|
||||
{
|
||||
if ($null -eq $pair.First -or $null -eq $pair.Return) {
|
||||
$failures.Add("$($pair.Label): missing same-location stationary sample")
|
||||
|
|
@ -273,8 +279,8 @@ function Add-RelativeGateFailures {
|
|||
$warnings.Add("$($pair.Label): connected animation population changed $($pair.First.AnimationCount) -> $($pair.Return.AnimationCount)")
|
||||
}
|
||||
|
||||
$privateLimit = [Math]::Max(384.0, $pair.First.PrivateMiB * 0.35)
|
||||
$workingLimit = [Math]::Max(384.0, $pair.First.WorkingSetMiB * 0.35)
|
||||
$privateLimit = [Math]::Max(192.0, $pair.First.PrivateMiB * 0.20)
|
||||
$workingLimit = [Math]::Max(192.0, $pair.First.WorkingSetMiB * 0.20)
|
||||
if (($pair.Return.PrivateMiB - $pair.First.PrivateMiB) -gt $privateLimit) {
|
||||
$failures.Add("$($pair.Label): private memory grew by $([Math]::Round($pair.Return.PrivateMiB - $pair.First.PrivateMiB, 1)) MiB")
|
||||
}
|
||||
|
|
@ -305,7 +311,9 @@ function Add-LogFailures {
|
|||
'WeenieError',
|
||||
'device removed',
|
||||
'GPU reset',
|
||||
'live: disconnected'
|
||||
'live: disconnected',
|
||||
'[shutdown]',
|
||||
'ObjectDisposedException'
|
||||
)
|
||||
foreach ($pattern in $fatalPatterns) {
|
||||
$count = (Get-PatternCount $stdoutLog $pattern) + (Get-PatternCount $stderrLog $pattern)
|
||||
|
|
@ -369,6 +377,7 @@ $env:ACDREAM_UNCAPPED_RENDER = '1'
|
|||
$env:ACDREAM_DEVTOOLS = '0'
|
||||
$env:ACDREAM_UI_PROBE_DUMP = '0'
|
||||
$env:ACDREAM_UI_PROBE_SCRIPT = $routePath
|
||||
$env:ACDREAM_AUTOMATION_ARTIFACT_DIR = $artifactDir
|
||||
$env:ACDREAM_DUMP_MOVE_TRUTH = '1'
|
||||
$env:ACDREAM_NO_AUDIO = $null
|
||||
$env:ACDREAM_WB_DIAG = $null
|
||||
|
|
@ -422,7 +431,6 @@ try {
|
|||
}
|
||||
|
||||
Add-RelativeGateFailures
|
||||
Add-LogFailures
|
||||
|
||||
Write-Marker 'route-complete requesting graceful close'
|
||||
$gracefulExit = Close-ClientGracefully $process
|
||||
|
|
@ -430,6 +438,7 @@ try {
|
|||
if ($process.HasExited) { $exitCode = [int]$process.ExitCode }
|
||||
if (-not $gracefulExit) { $failures.Add('client did not exit through the graceful WM_CLOSE path') }
|
||||
if ($null -ne $exitCode -and $exitCode -ne 0) { $failures.Add("client exit code was $exitCode") }
|
||||
Add-LogFailures
|
||||
}
|
||||
catch {
|
||||
$failures.Add($_.Exception.Message)
|
||||
|
|
|
|||
|
|
@ -113,6 +113,8 @@ function Add-LogFailures([string]$Label, [string]$Stdout, [string]$Stderr) {
|
|||
'device removed',
|
||||
'GPU reset',
|
||||
'live: disconnected',
|
||||
'[shutdown]',
|
||||
'ObjectDisposedException',
|
||||
'screenshot-failed',
|
||||
'graceful logout confirmation timed out',
|
||||
'graceful logout failed',
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue