From b03371c03df08895b76d30ffe2cc4d6ace91218f Mon Sep 17 00:00:00 2001 From: Erik Date: Mon, 20 Jul 2026 22:39:34 +0200 Subject: [PATCH] test(runtime): add connected lifecycle gate --- tools/connected-world-lifecycle.route.txt | 67 ++++ tools/connected-world-reconnect.route.txt | 12 + tools/run-connected-world-lifecycle-gate.ps1 | 335 +++++++++++++++++++ 3 files changed, 414 insertions(+) create mode 100644 tools/connected-world-lifecycle.route.txt create mode 100644 tools/connected-world-reconnect.route.txt create mode 100644 tools/run-connected-world-lifecycle-gate.ps1 diff --git a/tools/connected-world-lifecycle.route.txt b/tools/connected-world-lifecycle.route.txt new file mode 100644 index 00000000..72f019a7 --- /dev/null +++ b/tools/connected-world-lifecycle.route.txt @@ -0,0 +1,67 @@ +# Connected capped lifecycle route. Readiness and screenshots are driven by +# canonical runtime state; sleeps are only stationary sampling windows. + +wait world-ready 90000 +wait world-visible 30000 +sleep 30000 +checkpoint capped_login +screenshot capped_login 15000 + +# Dense outdoor island baseline. +command /teleloc 0x09040008 11.4 188.6 87.705 +wait materialized 1 90000 +wait world-visible 30000 +sleep 12000 +input down MovementTurnRight +sleep 6000 +input up MovementTurnRight +sleep 12000 +checkpoint aerlinthe_first +screenshot aerlinthe_first 15000 + +# World-edge streaming transition. +command /teleloc 0xF6820033 145.7 49.855 58.010 +wait materialized 2 90000 +wait world-visible 30000 +sleep 12000 +input down MovementTurnRight +sleep 6000 +input up MovementTurnRight +sleep 12000 +checkpoint rynthid + +# Facility Hub indoor cell: one-landblock collapse + exact EnvCell collision. +command /teleloc 0x8A020164 70.35 -40.66 -5.9 +wait materialized 3 90000 +wait world-visible 30000 +sleep 12000 +input down MovementTurnRight +sleep 6000 +input up MovementTurnRight +sleep 12000 +checkpoint facility_hub +screenshot facility_hub 15000 + +# Dungeon exit back to a normal outdoor near ring. +command /teleloc 0xA9B40019 84.0 7.1 94.005 +wait materialized 4 90000 +wait world-visible 30000 +sleep 12000 +input down MovementTurnRight +sleep 6000 +input up MovementTurnRight +sleep 12000 +checkpoint holtburg_after_dungeon +screenshot holtburg_after_dungeon 15000 + +# Same-location owner/resource comparison after outdoor/edge/dungeon churn. +command /teleloc 0x09040008 11.4 188.6 87.705 +wait materialized 5 90000 +wait world-visible 30000 +sleep 12000 +input down MovementTurnRight +sleep 6000 +input up MovementTurnRight +sleep 30000 +checkpoint aerlinthe_revisit +screenshot aerlinthe_revisit 15000 diff --git a/tools/connected-world-reconnect.route.txt b/tools/connected-world-reconnect.route.txt new file mode 100644 index 00000000..32f7dbf2 --- /dev/null +++ b/tools/connected-world-reconnect.route.txt @@ -0,0 +1,12 @@ +# Fresh-process reconnect gate. This session is uncapped so the same artifact +# records renderer throughput independently from the capped correctness run. + +wait world-ready 90000 +wait world-visible 30000 +sleep 30000 +input down MovementTurnRight +sleep 10000 +input up MovementTurnRight +sleep 10000 +checkpoint uncapped_reconnect +screenshot uncapped_reconnect 15000 diff --git a/tools/run-connected-world-lifecycle-gate.ps1 b/tools/run-connected-world-lifecycle-gate.ps1 new file mode 100644 index 00000000..861bd45b --- /dev/null +++ b/tools/run-connected-world-lifecycle-gate.ps1 @@ -0,0 +1,335 @@ +[CmdletBinding()] +param( + [string]$Repository = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path, + [string]$Account = $env:ACDREAM_TEST_USER, + [string]$Password = $env:ACDREAM_TEST_PASS, + [switch]$SkipBuild, + [int]$SessionTimeoutSeconds = 420 +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +if ([string]::IsNullOrWhiteSpace($Account)) { $Account = 'testaccount' } +if ([string]::IsNullOrWhiteSpace($Password)) { $Password = 'testpassword' } + +$stamp = Get-Date -Format 'yyyyMMdd-HHmmss' +$root = Join-Path $Repository "logs\connected-world-gate-$stamp" +$null = New-Item -ItemType Directory -Force -Path $root +$reportPath = Join-Path $root 'report.json' +$exe = Join-Path $Repository 'src\AcDream.App\bin\Release\net10.0\AcDream.App.exe' +$failures = [System.Collections.Generic.List[string]]::new() +$warnings = [System.Collections.Generic.List[string]]::new() +$sessions = [System.Collections.Generic.List[object]]::new() +$startedUtc = [DateTime]::UtcNow + +function Get-PatternCount([string]$Path, [string]$Pattern) { + if (-not (Test-Path -LiteralPath $Path)) { return 0 } + return @(Get-Content -LiteralPath $Path -ErrorAction SilentlyContinue | + Select-String -SimpleMatch $Pattern).Count +} + +function Wait-ForPattern( + [Diagnostics.Process]$Client, + [string]$Path, + [string]$Pattern, + [int]$TimeoutSeconds) +{ + $deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds) + while ([DateTime]::UtcNow -lt $deadline) { + $Client.Refresh() + if ($Client.HasExited) { + throw "client exited with code $($Client.ExitCode) while waiting for '$Pattern'" + } + if ((Get-PatternCount $Path $Pattern) -gt 0) { return } + Start-Sleep -Milliseconds 250 + } + throw "timed out after $TimeoutSeconds seconds waiting for '$Pattern'" +} + +function Close-ClientGracefully([Diagnostics.Process]$Client) { + $Client.Refresh() + if ($Client.HasExited) { return $true } + if (-not $Client.CloseMainWindow()) { return $false } + if (-not $Client.WaitForExit(30000)) { return $false } + $Client.WaitForExit() + return $true +} + +function Test-Png([string]$Path) { + if (-not (Test-Path -LiteralPath $Path)) { return $false } + $info = Get-Item -LiteralPath $Path + if ($info.Length -lt 1024) { return $false } + $bytes = [System.IO.File]::ReadAllBytes($Path) + if ($bytes.Length -lt 8) { return $false } + $signature = @(137, 80, 78, 71, 13, 10, 26, 10) + for ($i = 0; $i -lt $signature.Count; $i++) { + if ($bytes[$i] -ne $signature[$i]) { return $false } + } + return $true +} + +function Add-LogFailures([string]$Label, [string]$Stdout, [string]$Stderr) { + $fatalPatterns = @( + 'event=invariant-failure', + 'Unhandled exception', + 'AccessViolation', + 'OutOfMemoryException', + 'WeenieError', + 'device removed', + 'GPU reset', + 'live: disconnected', + 'screenshot-failed' + ) + foreach ($pattern in $fatalPatterns) { + $count = (Get-PatternCount $Stdout $pattern) + (Get-PatternCount $Stderr $pattern) + if ($count -gt 0) { $failures.Add("${Label}: '$pattern' appeared $count time(s)") } + } + + $missingLandblocks = Get-PatternCount $Stdout 'LandblockLoader.Load returned null' + if ($missingLandblocks -gt 0) { + $warnings.Add("${Label}: $missingLandblocks expected world-edge landblock miss(es)") + } +} + +function Read-Checkpoints([string]$Path) { + if (-not (Test-Path -LiteralPath $Path)) { return @() } + return @(Get-Content -LiteralPath $Path | ForEach-Object { $_ | ConvertFrom-Json }) +} + +function Validate-Checkpoint([string]$SessionLabel, [object]$Checkpoint) { + $name = $Checkpoint.name + $reveal = $Checkpoint.reveal + $resources = $Checkpoint.resources + if (-not $reveal.readiness.isReady) { + $failures.Add("${SessionLabel}/${name}: reveal was not ready") + } + if (-not $reveal.worldViewportObserved) { + $failures.Add("${SessionLabel}/${name}: normal world viewport was never observed") + } + if ($reveal.invariantFailureCount -ne 0) { + $failures.Add("${SessionLabel}/${name}: reveal has $($reveal.invariantFailureCount) invariant failure(s)") + } + if (-not $reveal.readiness.isUnhydratable) { + if (-not $reveal.readiness.isRenderNeighborhoodReady) { + $failures.Add("${SessionLabel}/${name}: render neighborhood was not ready") + } + if (-not $reveal.readiness.areCompositeTexturesReady) { + $failures.Add("${SessionLabel}/${name}: composite textures were not ready") + } + if (-not $reveal.readiness.isCollisionReady) { + $failures.Add("${SessionLabel}/${name}: collision was not ready") + } + } + if ($resources.pendingLiveTeardowns -ne 0) { + $failures.Add("${SessionLabel}/${name}: $($resources.pendingLiveTeardowns) live teardown(s) pending") + } + if ($resources.pendingLandblockRetirements -ne 0) { + $failures.Add("${SessionLabel}/${name}: $($resources.pendingLandblockRetirements) landblock retirement(s) pending") + } + if ($resources.stagedMeshUploads -ne 0) { + $failures.Add("${SessionLabel}/${name}: $($resources.stagedMeshUploads) staged mesh upload(s) remain at stable checkpoint") + } + if ($resources.compositeWarmupPending -ne 0) { + $failures.Add("${SessionLabel}/${name}: $($resources.compositeWarmupPending) composite warmup item(s) remain") + } + if ($resources.loadedLandblocks -le 0 -or $resources.worldEntities -le 0) { + $failures.Add("${SessionLabel}/${name}: world ownership is empty at a visible checkpoint") + } + if ($null -eq $resources.lastFrameProfile) { + $failures.Add("${SessionLabel}/${name}: no frame-profiler sample was available") + } +} + +function Invoke-Session( + [string]$Label, + [string]$RoutePath, + [bool]$Uncapped, + [string[]]$ExpectedCheckpoints, + [string[]]$ExpectedScreenshots) +{ + $sessionDir = Join-Path $root $Label + $artifactDir = Join-Path $sessionDir 'artifacts' + $null = New-Item -ItemType Directory -Force -Path $artifactDir + $stdout = Join-Path $sessionDir 'stdout.log' + $stderr = Join-Path $sessionDir 'stderr.log' + $timeline = Join-Path $artifactDir 'world-lifecycle.checkpoints.jsonl' + $client = $null + $graceful = $false + $exitCode = $null + $elapsed = [Diagnostics.Stopwatch]::StartNew() + + $env:ACDREAM_DAT_DIR = "$env:USERPROFILE\Documents\Asheron's Call" + $env:ACDREAM_LIVE = '1' + $env:ACDREAM_TEST_HOST = '127.0.0.1' + $env:ACDREAM_TEST_PORT = '9000' + $env:ACDREAM_TEST_USER = $Account + $env:ACDREAM_TEST_PASS = $Password + $env:ACDREAM_RETAIL_UI = '1' + $env:ACDREAM_FRAME_PROF = '1' + $env:ACDREAM_UNCAPPED_RENDER = if ($Uncapped) { '1' } else { $null } + $env:ACDREAM_DEVTOOLS = '0' + $env:ACDREAM_UI_PROBE_DUMP = '0' + $env:ACDREAM_UI_PROBE_SCRIPT = $RoutePath + $env:ACDREAM_AUTOMATION_ARTIFACT_DIR = $artifactDir + $env:ACDREAM_DUMP_MOVE_TRUTH = $null + $env:ACDREAM_WB_DIAG = $null + + try { + $client = Start-Process -FilePath $exe -WorkingDirectory $Repository ` + -RedirectStandardOutput $stdout -RedirectStandardError $stderr -PassThru + Wait-ForPattern $client $stdout '[UI-PROBE] UI probe script complete' $SessionTimeoutSeconds + + $client.Refresh() + $processSample = [pscustomobject][ordered]@{ + WorkingSetMiB = [Math]::Round($client.WorkingSet64 / 1MB, 1) + PrivateMiB = [Math]::Round($client.PrivateMemorySize64 / 1MB, 1) + HandleCount = $client.HandleCount + ThreadCount = $client.Threads.Count + WindowTitle = $client.MainWindowTitle + } + + $checkpoints = Read-Checkpoints $timeline + if ($checkpoints.Count -ne $ExpectedCheckpoints.Count) { + $failures.Add("${Label}: expected $($ExpectedCheckpoints.Count) checkpoints, found $($checkpoints.Count)") + } + foreach ($name in $ExpectedCheckpoints) { + $matches = @($checkpoints | Where-Object { $_.name -eq $name }) + if ($matches.Count -ne 1) { + $failures.Add("${Label}: expected one checkpoint '$name', found $($matches.Count)") + } + } + foreach ($checkpoint in $checkpoints) { Validate-Checkpoint $Label $checkpoint } + + foreach ($name in $ExpectedScreenshots) { + $png = Join-Path $artifactDir "screenshots\$name.png" + if (-not (Test-Png $png)) { $failures.Add("${Label}: missing or invalid screenshot '$png'") } + } + + Add-LogFailures $Label $stdout $stderr + $graceful = Close-ClientGracefully $client + $client.Refresh() + if ($client.HasExited) { $exitCode = [int]$client.ExitCode } + if (-not $graceful) { $failures.Add("${Label}: client did not close through WM_CLOSE") } + if ($null -ne $exitCode -and $exitCode -ne 0) { + $failures.Add("${Label}: client exited with code $exitCode") + } + + $session = [pscustomobject][ordered]@{ + Label = $Label + Uncapped = $Uncapped + ElapsedSeconds = [Math]::Round($elapsed.Elapsed.TotalSeconds, 3) + GracefulExit = $graceful + ExitCode = $exitCode + Process = $processSample + Checkpoints = @($checkpoints) + Stdout = $stdout + Stderr = $stderr + ArtifactDirectory = $artifactDir + } + $sessions.Add($session) + return $session + } + catch { + $failures.Add("${Label}: $($_.Exception.Message)") + return $null + } + finally { + if ($null -ne $client) { + $client.Refresh() + if (-not $client.HasExited) { + $graceful = Close-ClientGracefully $client + if (-not $graceful -and -not $client.HasExited) { + $failures.Add("${Label}: required forced termination after WM_CLOSE timeout") + Stop-Process -Id $client.Id -Force + $client.WaitForExit(10000) + } + } + $client.Dispose() + } + } +} + +function Add-SameLocationGates([object]$CappedSession) { + if ($null -eq $CappedSession) { return } + $first = @($CappedSession.Checkpoints | Where-Object { $_.name -eq 'aerlinthe_first' }) | Select-Object -First 1 + $revisit = @($CappedSession.Checkpoints | Where-Object { $_.name -eq 'aerlinthe_revisit' }) | Select-Object -First 1 + if ($null -eq $first -or $null -eq $revisit) { return } + + $managedLimit = [Math]::Max(256MB, [double]$first.resources.managedBytes * 0.40) + $gpuLimit = [Math]::Max(512MB, [double]$first.resources.trackedGpuBytes * 0.50) + if (($revisit.resources.managedBytes - $first.resources.managedBytes) -gt $managedLimit) { + $failures.Add("Aerlinthe revisit: managed memory grew beyond the connected gate tolerance") + } + if (($revisit.resources.trackedGpuBytes - $first.resources.trackedGpuBytes) -gt $gpuLimit) { + $failures.Add("Aerlinthe revisit: tracked GPU memory grew beyond the connected gate tolerance") + } + + foreach ($property in @('particleOwners', 'effectOwners', 'lightOwners', 'scriptOwners', 'compositeTextureOwners', 'particleTextureOwners')) { + $before = [double]$first.resources.$property + $after = [double]$revisit.resources.$property + $limit = [Math]::Max(64.0, $before * 0.50) + if (($after - $before) -gt $limit) { + $failures.Add("Aerlinthe revisit: owner '$property' grew $before -> $after") + } + } +} + +if (@(Get-Process -Name AcDream.App -ErrorAction SilentlyContinue).Count -gt 0) { + throw 'an AcDream.App client is already running; close it gracefully before the gate' +} +if (@(Get-NetUDPEndpoint -LocalPort 9000 -ErrorAction SilentlyContinue).Count -eq 0) { + throw 'local ACE is not listening on UDP port 9000' +} + +if (-not $SkipBuild) { + & dotnet build (Join-Path $Repository 'AcDream.slnx') -c Release --no-restore + if ($LASTEXITCODE -ne 0) { throw "Release build failed with exit code $LASTEXITCODE" } +} +if (-not (Test-Path -LiteralPath $exe)) { throw "client executable not found: $exe" } + +$capped = Invoke-Session ` + 'capped' ` + (Join-Path $Repository 'tools\connected-world-lifecycle.route.txt') ` + $false ` + @('capped_login', 'aerlinthe_first', 'rynthid', 'facility_hub', 'holtburg_after_dungeon', 'aerlinthe_revisit') ` + @('capped_login', 'aerlinthe_first', 'facility_hub', 'holtburg_after_dungeon', 'aerlinthe_revisit') + +Add-SameLocationGates $capped + +# The second process is both the session-teardown/reconnect gate and the +# uncapped renderer sample. A short pause lets ACE finish releasing the first +# session before the same account logs in again. +Start-Sleep -Seconds 3 +$null = Invoke-Session ` + 'uncapped-reconnect' ` + (Join-Path $Repository 'tools\connected-world-reconnect.route.txt') ` + $true ` + @('uncapped_reconnect') ` + @('uncapped_reconnect') + +$report = [pscustomobject][ordered]@{ + Passed = $failures.Count -eq 0 + StartedUtc = $startedUtc.ToString('O') + FinishedUtc = [DateTime]::UtcNow.ToString('O') + Commit = (& git -C $Repository rev-parse HEAD).Trim() + SourceStatus = @(& git -C $Repository status --short) + SessionName = $env:SESSIONNAME + VideoControllers = @(Get-CimInstance Win32_VideoController -ErrorAction SilentlyContinue | + ForEach-Object { [pscustomobject]@{ + Name = $_.Name + DriverVersion = $_.DriverVersion + AdapterRam = $_.AdapterRAM + } }) + Failures = @($failures) + Warnings = @($warnings) + Sessions = @($sessions) +} +$report | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $reportPath -Encoding utf8 + +Write-Output "REPORT=$reportPath" +Write-Output "RESULT=$(if ($report.Passed) { 'PASS' } else { 'FAIL' })" +foreach ($failure in $failures) { Write-Output "FAILURE=$failure" } +foreach ($warning in $warnings) { Write-Output "WARNING=$warning" } + +if ($failures.Count -gt 0) { exit 1 }