test(app): make canonical soak comparisons workload-aware

Pin every ACE teleloc to an explicit starting quaternion and distinguish stable-workload owner growth from legitimate authoritative population, visibility, and cache-retirement changes. Preserve the original process residency and frame-cost limits while making canonical deltas actionable.

Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
Erik 2026-07-22 20:13:29 +02:00
parent bca4148739
commit 6c5e0604c9
4 changed files with 88 additions and 44 deletions

View file

@ -136,17 +136,18 @@ Every checkpoint hard-gates zero:
- staged mesh uploads and bytes; - staged mesh uploads and bytes;
- composite warmup work. - composite warmup work.
The same-location Caul return → Caul plateau comparison requires exact equality Every route teleport supplies an explicit identity quaternion so the timed turn
for deterministic cache/allocator state: loaded/visible/total landblocks, begins from a reproducible heading. The same-location Caul return → Caul
teardown/retirement queues, mesh render-data/atlas/estimated bytes, staged work, plateau comparison requires exact equality for the deterministic loaded/total
tracked GPU bytes/buffers/textures, and composite/particle texture ownership. world domain, while visible-landblock and authoritative entity/animation/live-
Connected `WorldEntities`, `AnimatedEntities`, `LiveEntities`, and materialized entity changes are named workload warnings. Teardown/retirement queues, staged
live entities may legitimately change with authoritative population; those work, and composite warmup remain hard zero at every checkpoint. Mesh, atlas,
deltas are named workload warnings. VFX binding/owner/script/light counts remain tracked-GPU, composite/particle-texture, and VFX owner counts may retire and
hard equality gates when that population is stable, and become named workload shrink normally; any growth is a hard leak failure when the visible and
warnings when the authoritative population changed. Particles, emitters, and authoritative workload is stable, and a named owner-growth warning when that
active scripts are always transient workload warnings rather than silent workload changed. Particles, emitters, and active scripts are always transient
exclusions. workload warnings rather than silent exclusions. The report therefore says
which canonical owner grew without mistaking legitimate retirement for a leak.
Managed used/committed deltas are reported diagnostically. Existing Managed used/committed deltas are reported diagnostically. Existing
working/private-memory, update-p95, and allocation-p50 thresholds remain byte- working/private-memory, update-p95, and allocation-p50 thresholds remain byte-
for-byte unchanged as the secondary residency/performance guard. for-byte unchanged as the secondary residency/performance guard.

View file

@ -30,6 +30,21 @@ public sealed class ConnectedR6SoakContractTests
.ToArray(); .ToArray();
Assert.Equal(ExpectedCheckpointNames, checkpoints); Assert.Equal(ExpectedCheckpointNames, checkpoints);
string[] teleports = File.ReadAllLines(Path.Combine(
FindRepoRoot(),
"tools",
"connected-r6-soak.route.txt"))
.Select(static line => line.Trim())
.Where(static line => line.StartsWith(
"command /teleloc ",
StringComparison.Ordinal))
.ToArray();
Assert.Equal(9, teleports.Length);
Assert.All(teleports, static line => Assert.EndsWith(
" 1 0 0 0",
line,
StringComparison.Ordinal));
} }
[Fact] [Fact]
@ -67,6 +82,14 @@ public sealed class ConnectedR6SoakContractTests
"missing render-world synchronization fields", "missing render-world synchronization fields",
source, source,
StringComparison.Ordinal); StringComparison.Ordinal);
Assert.Contains(
"canonical cache '$field' grew",
source,
StringComparison.Ordinal);
Assert.Contains(
"if ($workloadChanged) { $warnings.Add($message) }",
source,
StringComparison.Ordinal);
string[] zeroFields = string[] zeroFields =
[ [

View file

@ -4,8 +4,9 @@
sleep 15000 sleep 15000
# Caul baseline + local movement/jump/combat exercise. # Caul baseline + local movement/jump/combat exercise. Every teleloc carries
command /teleloc 0xC95B0001 14.8 0.3 12.005 # the identity quaternion so each timed turn begins from the same heading.
command /teleloc 0xC95B0001 14.8 0.3 12.005 1 0 0 0
wait materialized 1 60000 wait materialized 1 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight
@ -43,7 +44,7 @@ sleep 7400
checkpoint caul-baseline checkpoint caul-baseline
# Sawato baseline. # Sawato baseline.
command /teleloc 0x3032001C 83.3 89.2 133.005 command /teleloc 0x3032001C 83.3 89.2 133.005 1 0 0 0
wait materialized 2 60000 wait materialized 2 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight
@ -53,7 +54,7 @@ sleep 22000
checkpoint sawato-baseline checkpoint sawato-baseline
# Rynthid world-edge streaming. # Rynthid world-edge streaming.
command /teleloc 0xF6820033 145.7 49.855 58.010 command /teleloc 0xF6820033 145.7 49.855 58.010 1 0 0 0
wait materialized 3 60000 wait materialized 3 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight
@ -63,7 +64,7 @@ sleep 22000
checkpoint rynthid checkpoint rynthid
# Aerlinthe dense island. # Aerlinthe dense island.
command /teleloc 0x09040008 11.4 188.6 87.705 command /teleloc 0x09040008 11.4 188.6 87.705 1 0 0 0
wait materialized 4 60000 wait materialized 4 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight
@ -73,7 +74,7 @@ sleep 22000
checkpoint aerlinthe checkpoint aerlinthe
# Same-location lifecycle revisit. # Same-location lifecycle revisit.
command /teleloc 0x3032001C 83.3 89.2 133.005 command /teleloc 0x3032001C 83.3 89.2 133.005 1 0 0 0
wait materialized 5 60000 wait materialized 5 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight
@ -83,7 +84,7 @@ sleep 22000
checkpoint sawato-return checkpoint sawato-return
# Holtburg mixed-town workload + local exercise. # Holtburg mixed-town workload + local exercise.
command /teleloc 0xA9B40019 84.0 7.1 94.005 command /teleloc 0xA9B40019 84.0 7.1 94.005 1 0 0 0
wait materialized 6 60000 wait materialized 6 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight
@ -121,7 +122,7 @@ sleep 7400
checkpoint holtburg checkpoint holtburg
# Caul return/leak oracle + final local exercise. # Caul return/leak oracle + final local exercise.
command /teleloc 0xC95B0001 14.8 0.3 12.005 command /teleloc 0xC95B0001 14.8 0.3 12.005 1 0 0 0
wait materialized 7 60000 wait materialized 7 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight
@ -162,7 +163,7 @@ checkpoint caul-return
# circuit intentionally warms bounded DAT/GPU caches, so comparing the cold # circuit intentionally warms bounded DAT/GPU caches, so comparing the cold
# first Caul against its warm return would punish successful source-window # first Caul against its warm return would punish successful source-window
# retirement. This circuit loads no new destination content. # retirement. This circuit loads no new destination content.
command /teleloc 0x3032001C 83.3 89.2 133.005 command /teleloc 0x3032001C 83.3 89.2 133.005 1 0 0 0
wait materialized 8 60000 wait materialized 8 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight
@ -171,7 +172,7 @@ input up MovementTurnRight
sleep 22000 sleep 22000
checkpoint sawato-plateau checkpoint sawato-plateau
command /teleloc 0xC95B0001 14.8 0.3 12.005 command /teleloc 0xC95B0001 14.8 0.3 12.005 1 0 0 0
wait materialized 9 60000 wait materialized 9 60000
sleep 8000 sleep 8000
input down MovementTurnRight input down MovementTurnRight

View file

@ -386,34 +386,30 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
foreach ($field in @( foreach ($field in @(
'loadedLandblocks', 'loadedLandblocks',
'visibleLandblocks', 'totalLandblocks')) {
'totalLandblocks',
'pendingLiveTeardowns',
'pendingLandblockRetirements',
'meshRenderData',
'meshAtlasArrays',
'meshEstimatedBytes',
'stagedMeshUploads',
'stagedMeshBytes',
'trackedGpuBytes',
'trackedGpuBuffers',
'trackedGpuTextures',
'ownedCompositeTextures',
'compositeTextureOwners',
'activeParticleTextures',
'particleTextureOwners',
'compositeWarmupPending')) {
$first = $caulReturn.resources.PSObject.Properties[$field] $first = $caulReturn.resources.PSObject.Properties[$field]
$plateau = $caulPlateau.resources.PSObject.Properties[$field] $plateau = $caulPlateau.resources.PSObject.Properties[$field]
if ($null -eq $first -or $null -eq $plateau) { if ($null -eq $first -or $null -eq $plateau) {
$failures.Add("canonical Caul comparison is missing resources.$field") $failures.Add("canonical Caul comparison is missing resources.$field")
} }
elseif ([long]$first.Value -ne [long]$plateau.Value) { elseif ([long]$first.Value -ne [long]$plateau.Value) {
$failures.Add("Caul plateau canonical owner '$field' changed $($first.Value) -> $($plateau.Value)") $failures.Add("Caul plateau deterministic world domain '$field' changed $($first.Value) -> $($plateau.Value)")
} }
} }
$populationChanged = $false $workloadChanged = $false
$firstVisible =
$caulReturn.resources.PSObject.Properties['visibleLandblocks']
$plateauVisible =
$caulPlateau.resources.PSObject.Properties['visibleLandblocks']
if ($null -eq $firstVisible -or $null -eq $plateauVisible) {
$failures.Add('canonical Caul workload comparison is missing resources.visibleLandblocks')
}
elseif ([long]$firstVisible.Value -ne [long]$plateauVisible.Value) {
$workloadChanged = $true
$warnings.Add("Caul plateau visible workload changed $($firstVisible.Value) -> $($plateauVisible.Value)")
}
foreach ($field in @( foreach ($field in @(
'worldEntities', 'worldEntities',
'animatedEntities', 'animatedEntities',
@ -425,11 +421,34 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
$failures.Add("canonical Caul population comparison is missing resources.$field") $failures.Add("canonical Caul population comparison is missing resources.$field")
} }
elseif ([long]$first.Value -ne [long]$plateau.Value) { elseif ([long]$first.Value -ne [long]$plateau.Value) {
$populationChanged = $true $workloadChanged = $true
$warnings.Add("Caul plateau authoritative population '$field' changed $($first.Value) -> $($plateau.Value)") $warnings.Add("Caul plateau authoritative population '$field' changed $($first.Value) -> $($plateau.Value)")
} }
} }
foreach ($field in @(
'meshRenderData',
'meshAtlasArrays',
'meshEstimatedBytes',
'trackedGpuBytes',
'trackedGpuBuffers',
'trackedGpuTextures',
'ownedCompositeTextures',
'compositeTextureOwners',
'activeParticleTextures',
'particleTextureOwners')) {
$first = $caulReturn.resources.PSObject.Properties[$field]
$plateau = $caulPlateau.resources.PSObject.Properties[$field]
if ($null -eq $first -or $null -eq $plateau) {
$failures.Add("canonical Caul cache comparison is missing resources.$field")
}
elseif ([long]$plateau.Value -gt [long]$first.Value) {
$message = "Caul plateau canonical cache '$field' grew $($first.Value) -> $($plateau.Value)"
if ($workloadChanged) { $warnings.Add($message) }
else { $failures.Add($message) }
}
}
foreach ($field in @( foreach ($field in @(
'particleBindings', 'particleBindings',
'particleOwners', 'particleOwners',
@ -441,9 +460,9 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
if ($null -eq $first -or $null -eq $plateau) { if ($null -eq $first -or $null -eq $plateau) {
$failures.Add("canonical Caul owner comparison is missing resources.$field") $failures.Add("canonical Caul owner comparison is missing resources.$field")
} }
elseif ([long]$first.Value -ne [long]$plateau.Value) { elseif ([long]$plateau.Value -gt [long]$first.Value) {
$message = "Caul plateau population-sensitive owner '$field' changed $($first.Value) -> $($plateau.Value)" $message = "Caul plateau population-sensitive owner '$field' grew $($first.Value) -> $($plateau.Value)"
if ($populationChanged) { $warnings.Add($message) } if ($workloadChanged) { $warnings.Add($message) }
else { $failures.Add($message) } else { $failures.Add($message) }
} }
} }