test(runtime): add connected lifecycle gate
This commit is contained in:
Erik
parent
354c2adc2e
commit
b03371c03d
3 changed files with 414 additions and 0 deletions
67
tools/connected-world-lifecycle.route.txt
Normal file
67
tools/connected-world-lifecycle.route.txt
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
# Connected capped lifecycle route. Readiness and screenshots are driven by
|
||||
# canonical runtime state; sleeps are only stationary sampling windows.
|
||||
|
||||
wait world-ready 90000
|
||||
wait world-visible 30000
|
||||
sleep 30000
|
||||
checkpoint capped_login
|
||||
screenshot capped_login 15000
|
||||
|
||||
# Dense outdoor island baseline.
|
||||
command /teleloc 0x09040008 11.4 188.6 87.705
|
||||
wait materialized 1 90000
|
||||
wait world-visible 30000
|
||||
sleep 12000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
input up MovementTurnRight
|
||||
sleep 12000
|
||||
checkpoint aerlinthe_first
|
||||
screenshot aerlinthe_first 15000
|
||||
|
||||
# World-edge streaming transition.
|
||||
command /teleloc 0xF6820033 145.7 49.855 58.010
|
||||
wait materialized 2 90000
|
||||
wait world-visible 30000
|
||||
sleep 12000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
input up MovementTurnRight
|
||||
sleep 12000
|
||||
checkpoint rynthid
|
||||
|
||||
# Facility Hub indoor cell: one-landblock collapse + exact EnvCell collision.
|
||||
command /teleloc 0x8A020164 70.35 -40.66 -5.9
|
||||
wait materialized 3 90000
|
||||
wait world-visible 30000
|
||||
sleep 12000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
input up MovementTurnRight
|
||||
sleep 12000
|
||||
checkpoint facility_hub
|
||||
screenshot facility_hub 15000
|
||||
|
||||
# Dungeon exit back to a normal outdoor near ring.
|
||||
command /teleloc 0xA9B40019 84.0 7.1 94.005
|
||||
wait materialized 4 90000
|
||||
wait world-visible 30000
|
||||
sleep 12000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
input up MovementTurnRight
|
||||
sleep 12000
|
||||
checkpoint holtburg_after_dungeon
|
||||
screenshot holtburg_after_dungeon 15000
|
||||
|
||||
# Same-location owner/resource comparison after outdoor/edge/dungeon churn.
|
||||
command /teleloc 0x09040008 11.4 188.6 87.705
|
||||
wait materialized 5 90000
|
||||
wait world-visible 30000
|
||||
sleep 12000
|
||||
input down MovementTurnRight
|
||||
sleep 6000
|
||||
input up MovementTurnRight
|
||||
sleep 30000
|
||||
checkpoint aerlinthe_revisit
|
||||
screenshot aerlinthe_revisit 15000
|
||||
12
tools/connected-world-reconnect.route.txt
Normal file
12
tools/connected-world-reconnect.route.txt
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# Fresh-process reconnect gate. This session is uncapped so the same artifact
|
||||
# records renderer throughput independently from the capped correctness run.
|
||||
|
||||
wait world-ready 90000
|
||||
wait world-visible 30000
|
||||
sleep 30000
|
||||
input down MovementTurnRight
|
||||
sleep 10000
|
||||
input up MovementTurnRight
|
||||
sleep 10000
|
||||
checkpoint uncapped_reconnect
|
||||
screenshot uncapped_reconnect 15000
|
||||
335
tools/run-connected-world-lifecycle-gate.ps1
Normal file
335
tools/run-connected-world-lifecycle-gate.ps1
Normal file
|
|
@ -0,0 +1,335 @@
|
|||
[CmdletBinding()]
|
||||
param(
|
||||
[string]$Repository = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path,
|
||||
[string]$Account = $env:ACDREAM_TEST_USER,
|
||||
[string]$Password = $env:ACDREAM_TEST_PASS,
|
||||
[switch]$SkipBuild,
|
||||
[int]$SessionTimeoutSeconds = 420
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
if ([string]::IsNullOrWhiteSpace($Account)) { $Account = 'testaccount' }
|
||||
if ([string]::IsNullOrWhiteSpace($Password)) { $Password = 'testpassword' }
|
||||
|
||||
$stamp = Get-Date -Format 'yyyyMMdd-HHmmss'
|
||||
$root = Join-Path $Repository "logs\connected-world-gate-$stamp"
|
||||
$null = New-Item -ItemType Directory -Force -Path $root
|
||||
$reportPath = Join-Path $root 'report.json'
|
||||
$exe = Join-Path $Repository 'src\AcDream.App\bin\Release\net10.0\AcDream.App.exe'
|
||||
$failures = [System.Collections.Generic.List[string]]::new()
|
||||
$warnings = [System.Collections.Generic.List[string]]::new()
|
||||
$sessions = [System.Collections.Generic.List[object]]::new()
|
||||
$startedUtc = [DateTime]::UtcNow
|
||||
|
||||
function Get-PatternCount([string]$Path, [string]$Pattern) {
|
||||
if (-not (Test-Path -LiteralPath $Path)) { return 0 }
|
||||
return @(Get-Content -LiteralPath $Path -ErrorAction SilentlyContinue |
|
||||
Select-String -SimpleMatch $Pattern).Count
|
||||
}
|
||||
|
||||
function Wait-ForPattern(
|
||||
[Diagnostics.Process]$Client,
|
||||
[string]$Path,
|
||||
[string]$Pattern,
|
||||
[int]$TimeoutSeconds)
|
||||
{
|
||||
$deadline = [DateTime]::UtcNow.AddSeconds($TimeoutSeconds)
|
||||
while ([DateTime]::UtcNow -lt $deadline) {
|
||||
$Client.Refresh()
|
||||
if ($Client.HasExited) {
|
||||
throw "client exited with code $($Client.ExitCode) while waiting for '$Pattern'"
|
||||
}
|
||||
if ((Get-PatternCount $Path $Pattern) -gt 0) { return }
|
||||
Start-Sleep -Milliseconds 250
|
||||
}
|
||||
throw "timed out after $TimeoutSeconds seconds waiting for '$Pattern'"
|
||||
}
|
||||
|
||||
function Close-ClientGracefully([Diagnostics.Process]$Client) {
|
||||
$Client.Refresh()
|
||||
if ($Client.HasExited) { return $true }
|
||||
if (-not $Client.CloseMainWindow()) { return $false }
|
||||
if (-not $Client.WaitForExit(30000)) { return $false }
|
||||
$Client.WaitForExit()
|
||||
return $true
|
||||
}
|
||||
|
||||
function Test-Png([string]$Path) {
|
||||
if (-not (Test-Path -LiteralPath $Path)) { return $false }
|
||||
$info = Get-Item -LiteralPath $Path
|
||||
if ($info.Length -lt 1024) { return $false }
|
||||
$bytes = [System.IO.File]::ReadAllBytes($Path)
|
||||
if ($bytes.Length -lt 8) { return $false }
|
||||
$signature = @(137, 80, 78, 71, 13, 10, 26, 10)
|
||||
for ($i = 0; $i -lt $signature.Count; $i++) {
|
||||
if ($bytes[$i] -ne $signature[$i]) { return $false }
|
||||
}
|
||||
return $true
|
||||
}
|
||||
|
||||
function Add-LogFailures([string]$Label, [string]$Stdout, [string]$Stderr) {
|
||||
$fatalPatterns = @(
|
||||
'event=invariant-failure',
|
||||
'Unhandled exception',
|
||||
'AccessViolation',
|
||||
'OutOfMemoryException',
|
||||
'WeenieError',
|
||||
'device removed',
|
||||
'GPU reset',
|
||||
'live: disconnected',
|
||||
'screenshot-failed'
|
||||
)
|
||||
foreach ($pattern in $fatalPatterns) {
|
||||
$count = (Get-PatternCount $Stdout $pattern) + (Get-PatternCount $Stderr $pattern)
|
||||
if ($count -gt 0) { $failures.Add("${Label}: '$pattern' appeared $count time(s)") }
|
||||
}
|
||||
|
||||
$missingLandblocks = Get-PatternCount $Stdout 'LandblockLoader.Load returned null'
|
||||
if ($missingLandblocks -gt 0) {
|
||||
$warnings.Add("${Label}: $missingLandblocks expected world-edge landblock miss(es)")
|
||||
}
|
||||
}
|
||||
|
||||
function Read-Checkpoints([string]$Path) {
|
||||
if (-not (Test-Path -LiteralPath $Path)) { return @() }
|
||||
return @(Get-Content -LiteralPath $Path | ForEach-Object { $_ | ConvertFrom-Json })
|
||||
}
|
||||
|
||||
function Validate-Checkpoint([string]$SessionLabel, [object]$Checkpoint) {
|
||||
$name = $Checkpoint.name
|
||||
$reveal = $Checkpoint.reveal
|
||||
$resources = $Checkpoint.resources
|
||||
if (-not $reveal.readiness.isReady) {
|
||||
$failures.Add("${SessionLabel}/${name}: reveal was not ready")
|
||||
}
|
||||
if (-not $reveal.worldViewportObserved) {
|
||||
$failures.Add("${SessionLabel}/${name}: normal world viewport was never observed")
|
||||
}
|
||||
if ($reveal.invariantFailureCount -ne 0) {
|
||||
$failures.Add("${SessionLabel}/${name}: reveal has $($reveal.invariantFailureCount) invariant failure(s)")
|
||||
}
|
||||
if (-not $reveal.readiness.isUnhydratable) {
|
||||
if (-not $reveal.readiness.isRenderNeighborhoodReady) {
|
||||
$failures.Add("${SessionLabel}/${name}: render neighborhood was not ready")
|
||||
}
|
||||
if (-not $reveal.readiness.areCompositeTexturesReady) {
|
||||
$failures.Add("${SessionLabel}/${name}: composite textures were not ready")
|
||||
}
|
||||
if (-not $reveal.readiness.isCollisionReady) {
|
||||
$failures.Add("${SessionLabel}/${name}: collision was not ready")
|
||||
}
|
||||
}
|
||||
if ($resources.pendingLiveTeardowns -ne 0) {
|
||||
$failures.Add("${SessionLabel}/${name}: $($resources.pendingLiveTeardowns) live teardown(s) pending")
|
||||
}
|
||||
if ($resources.pendingLandblockRetirements -ne 0) {
|
||||
$failures.Add("${SessionLabel}/${name}: $($resources.pendingLandblockRetirements) landblock retirement(s) pending")
|
||||
}
|
||||
if ($resources.stagedMeshUploads -ne 0) {
|
||||
$failures.Add("${SessionLabel}/${name}: $($resources.stagedMeshUploads) staged mesh upload(s) remain at stable checkpoint")
|
||||
}
|
||||
if ($resources.compositeWarmupPending -ne 0) {
|
||||
$failures.Add("${SessionLabel}/${name}: $($resources.compositeWarmupPending) composite warmup item(s) remain")
|
||||
}
|
||||
if ($resources.loadedLandblocks -le 0 -or $resources.worldEntities -le 0) {
|
||||
$failures.Add("${SessionLabel}/${name}: world ownership is empty at a visible checkpoint")
|
||||
}
|
||||
if ($null -eq $resources.lastFrameProfile) {
|
||||
$failures.Add("${SessionLabel}/${name}: no frame-profiler sample was available")
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Session(
|
||||
[string]$Label,
|
||||
[string]$RoutePath,
|
||||
[bool]$Uncapped,
|
||||
[string[]]$ExpectedCheckpoints,
|
||||
[string[]]$ExpectedScreenshots)
|
||||
{
|
||||
$sessionDir = Join-Path $root $Label
|
||||
$artifactDir = Join-Path $sessionDir 'artifacts'
|
||||
$null = New-Item -ItemType Directory -Force -Path $artifactDir
|
||||
$stdout = Join-Path $sessionDir 'stdout.log'
|
||||
$stderr = Join-Path $sessionDir 'stderr.log'
|
||||
$timeline = Join-Path $artifactDir 'world-lifecycle.checkpoints.jsonl'
|
||||
$client = $null
|
||||
$graceful = $false
|
||||
$exitCode = $null
|
||||
$elapsed = [Diagnostics.Stopwatch]::StartNew()
|
||||
|
||||
$env:ACDREAM_DAT_DIR = "$env:USERPROFILE\Documents\Asheron's Call"
|
||||
$env:ACDREAM_LIVE = '1'
|
||||
$env:ACDREAM_TEST_HOST = '127.0.0.1'
|
||||
$env:ACDREAM_TEST_PORT = '9000'
|
||||
$env:ACDREAM_TEST_USER = $Account
|
||||
$env:ACDREAM_TEST_PASS = $Password
|
||||
$env:ACDREAM_RETAIL_UI = '1'
|
||||
$env:ACDREAM_FRAME_PROF = '1'
|
||||
$env:ACDREAM_UNCAPPED_RENDER = if ($Uncapped) { '1' } else { $null }
|
||||
$env:ACDREAM_DEVTOOLS = '0'
|
||||
$env:ACDREAM_UI_PROBE_DUMP = '0'
|
||||
$env:ACDREAM_UI_PROBE_SCRIPT = $RoutePath
|
||||
$env:ACDREAM_AUTOMATION_ARTIFACT_DIR = $artifactDir
|
||||
$env:ACDREAM_DUMP_MOVE_TRUTH = $null
|
||||
$env:ACDREAM_WB_DIAG = $null
|
||||
|
||||
try {
|
||||
$client = Start-Process -FilePath $exe -WorkingDirectory $Repository `
|
||||
-RedirectStandardOutput $stdout -RedirectStandardError $stderr -PassThru
|
||||
Wait-ForPattern $client $stdout '[UI-PROBE] UI probe script complete' $SessionTimeoutSeconds
|
||||
|
||||
$client.Refresh()
|
||||
$processSample = [pscustomobject][ordered]@{
|
||||
WorkingSetMiB = [Math]::Round($client.WorkingSet64 / 1MB, 1)
|
||||
PrivateMiB = [Math]::Round($client.PrivateMemorySize64 / 1MB, 1)
|
||||
HandleCount = $client.HandleCount
|
||||
ThreadCount = $client.Threads.Count
|
||||
WindowTitle = $client.MainWindowTitle
|
||||
}
|
||||
|
||||
$checkpoints = Read-Checkpoints $timeline
|
||||
if ($checkpoints.Count -ne $ExpectedCheckpoints.Count) {
|
||||
$failures.Add("${Label}: expected $($ExpectedCheckpoints.Count) checkpoints, found $($checkpoints.Count)")
|
||||
}
|
||||
foreach ($name in $ExpectedCheckpoints) {
|
||||
$matches = @($checkpoints | Where-Object { $_.name -eq $name })
|
||||
if ($matches.Count -ne 1) {
|
||||
$failures.Add("${Label}: expected one checkpoint '$name', found $($matches.Count)")
|
||||
}
|
||||
}
|
||||
foreach ($checkpoint in $checkpoints) { Validate-Checkpoint $Label $checkpoint }
|
||||
|
||||
foreach ($name in $ExpectedScreenshots) {
|
||||
$png = Join-Path $artifactDir "screenshots\$name.png"
|
||||
if (-not (Test-Png $png)) { $failures.Add("${Label}: missing or invalid screenshot '$png'") }
|
||||
}
|
||||
|
||||
Add-LogFailures $Label $stdout $stderr
|
||||
$graceful = Close-ClientGracefully $client
|
||||
$client.Refresh()
|
||||
if ($client.HasExited) { $exitCode = [int]$client.ExitCode }
|
||||
if (-not $graceful) { $failures.Add("${Label}: client did not close through WM_CLOSE") }
|
||||
if ($null -ne $exitCode -and $exitCode -ne 0) {
|
||||
$failures.Add("${Label}: client exited with code $exitCode")
|
||||
}
|
||||
|
||||
$session = [pscustomobject][ordered]@{
|
||||
Label = $Label
|
||||
Uncapped = $Uncapped
|
||||
ElapsedSeconds = [Math]::Round($elapsed.Elapsed.TotalSeconds, 3)
|
||||
GracefulExit = $graceful
|
||||
ExitCode = $exitCode
|
||||
Process = $processSample
|
||||
Checkpoints = @($checkpoints)
|
||||
Stdout = $stdout
|
||||
Stderr = $stderr
|
||||
ArtifactDirectory = $artifactDir
|
||||
}
|
||||
$sessions.Add($session)
|
||||
return $session
|
||||
}
|
||||
catch {
|
||||
$failures.Add("${Label}: $($_.Exception.Message)")
|
||||
return $null
|
||||
}
|
||||
finally {
|
||||
if ($null -ne $client) {
|
||||
$client.Refresh()
|
||||
if (-not $client.HasExited) {
|
||||
$graceful = Close-ClientGracefully $client
|
||||
if (-not $graceful -and -not $client.HasExited) {
|
||||
$failures.Add("${Label}: required forced termination after WM_CLOSE timeout")
|
||||
Stop-Process -Id $client.Id -Force
|
||||
$client.WaitForExit(10000)
|
||||
}
|
||||
}
|
||||
$client.Dispose()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Add-SameLocationGates([object]$CappedSession) {
|
||||
if ($null -eq $CappedSession) { return }
|
||||
$first = @($CappedSession.Checkpoints | Where-Object { $_.name -eq 'aerlinthe_first' }) | Select-Object -First 1
|
||||
$revisit = @($CappedSession.Checkpoints | Where-Object { $_.name -eq 'aerlinthe_revisit' }) | Select-Object -First 1
|
||||
if ($null -eq $first -or $null -eq $revisit) { return }
|
||||
|
||||
$managedLimit = [Math]::Max(256MB, [double]$first.resources.managedBytes * 0.40)
|
||||
$gpuLimit = [Math]::Max(512MB, [double]$first.resources.trackedGpuBytes * 0.50)
|
||||
if (($revisit.resources.managedBytes - $first.resources.managedBytes) -gt $managedLimit) {
|
||||
$failures.Add("Aerlinthe revisit: managed memory grew beyond the connected gate tolerance")
|
||||
}
|
||||
if (($revisit.resources.trackedGpuBytes - $first.resources.trackedGpuBytes) -gt $gpuLimit) {
|
||||
$failures.Add("Aerlinthe revisit: tracked GPU memory grew beyond the connected gate tolerance")
|
||||
}
|
||||
|
||||
foreach ($property in @('particleOwners', 'effectOwners', 'lightOwners', 'scriptOwners', 'compositeTextureOwners', 'particleTextureOwners')) {
|
||||
$before = [double]$first.resources.$property
|
||||
$after = [double]$revisit.resources.$property
|
||||
$limit = [Math]::Max(64.0, $before * 0.50)
|
||||
if (($after - $before) -gt $limit) {
|
||||
$failures.Add("Aerlinthe revisit: owner '$property' grew $before -> $after")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (@(Get-Process -Name AcDream.App -ErrorAction SilentlyContinue).Count -gt 0) {
|
||||
throw 'an AcDream.App client is already running; close it gracefully before the gate'
|
||||
}
|
||||
if (@(Get-NetUDPEndpoint -LocalPort 9000 -ErrorAction SilentlyContinue).Count -eq 0) {
|
||||
throw 'local ACE is not listening on UDP port 9000'
|
||||
}
|
||||
|
||||
if (-not $SkipBuild) {
|
||||
& dotnet build (Join-Path $Repository 'AcDream.slnx') -c Release --no-restore
|
||||
if ($LASTEXITCODE -ne 0) { throw "Release build failed with exit code $LASTEXITCODE" }
|
||||
}
|
||||
if (-not (Test-Path -LiteralPath $exe)) { throw "client executable not found: $exe" }
|
||||
|
||||
$capped = Invoke-Session `
|
||||
'capped' `
|
||||
(Join-Path $Repository 'tools\connected-world-lifecycle.route.txt') `
|
||||
$false `
|
||||
@('capped_login', 'aerlinthe_first', 'rynthid', 'facility_hub', 'holtburg_after_dungeon', 'aerlinthe_revisit') `
|
||||
@('capped_login', 'aerlinthe_first', 'facility_hub', 'holtburg_after_dungeon', 'aerlinthe_revisit')
|
||||
|
||||
Add-SameLocationGates $capped
|
||||
|
||||
# The second process is both the session-teardown/reconnect gate and the
|
||||
# uncapped renderer sample. A short pause lets ACE finish releasing the first
|
||||
# session before the same account logs in again.
|
||||
Start-Sleep -Seconds 3
|
||||
$null = Invoke-Session `
|
||||
'uncapped-reconnect' `
|
||||
(Join-Path $Repository 'tools\connected-world-reconnect.route.txt') `
|
||||
$true `
|
||||
@('uncapped_reconnect') `
|
||||
@('uncapped_reconnect')
|
||||
|
||||
$report = [pscustomobject][ordered]@{
|
||||
Passed = $failures.Count -eq 0
|
||||
StartedUtc = $startedUtc.ToString('O')
|
||||
FinishedUtc = [DateTime]::UtcNow.ToString('O')
|
||||
Commit = (& git -C $Repository rev-parse HEAD).Trim()
|
||||
SourceStatus = @(& git -C $Repository status --short)
|
||||
SessionName = $env:SESSIONNAME
|
||||
VideoControllers = @(Get-CimInstance Win32_VideoController -ErrorAction SilentlyContinue |
|
||||
ForEach-Object { [pscustomobject]@{
|
||||
Name = $_.Name
|
||||
DriverVersion = $_.DriverVersion
|
||||
AdapterRam = $_.AdapterRAM
|
||||
} })
|
||||
Failures = @($failures)
|
||||
Warnings = @($warnings)
|
||||
Sessions = @($sessions)
|
||||
}
|
||||
$report | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $reportPath -Encoding utf8
|
||||
|
||||
Write-Output "REPORT=$reportPath"
|
||||
Write-Output "RESULT=$(if ($report.Passed) { 'PASS' } else { 'FAIL' })"
|
||||
foreach ($failure in $failures) { Write-Output "FAILURE=$failure" }
|
||||
foreach ($warning in $warnings) { Write-Output "WARNING=$warning" }
|
||||
|
||||
if ($failures.Count -gt 0) { exit 1 }
|
||||
Loading…
Add table
Add a link
Reference in a new issue