test(app): make canonical soak comparisons workload-aware
Pin every ACE teleloc to an explicit starting quaternion and distinguish stable-workload owner growth from legitimate authoritative population, visibility, and cache-retirement changes. Preserve the original process residency and frame-cost limits while making canonical deltas actionable. Co-authored-by: Codex <codex@openai.com>
This commit is contained in:
parent
bca4148739
commit
6c5e0604c9
4 changed files with 88 additions and 44 deletions
|
|
@ -136,17 +136,18 @@ Every checkpoint hard-gates zero:
|
||||||
- staged mesh uploads and bytes;
|
- staged mesh uploads and bytes;
|
||||||
- composite warmup work.
|
- composite warmup work.
|
||||||
|
|
||||||
The same-location Caul return → Caul plateau comparison requires exact equality
|
Every route teleport supplies an explicit identity quaternion so the timed turn
|
||||||
for deterministic cache/allocator state: loaded/visible/total landblocks,
|
begins from a reproducible heading. The same-location Caul return → Caul
|
||||||
teardown/retirement queues, mesh render-data/atlas/estimated bytes, staged work,
|
plateau comparison requires exact equality for the deterministic loaded/total
|
||||||
tracked GPU bytes/buffers/textures, and composite/particle texture ownership.
|
world domain, while visible-landblock and authoritative entity/animation/live-
|
||||||
Connected `WorldEntities`, `AnimatedEntities`, `LiveEntities`, and materialized
|
entity changes are named workload warnings. Teardown/retirement queues, staged
|
||||||
live entities may legitimately change with authoritative population; those
|
work, and composite warmup remain hard zero at every checkpoint. Mesh, atlas,
|
||||||
deltas are named workload warnings. VFX binding/owner/script/light counts remain
|
tracked-GPU, composite/particle-texture, and VFX owner counts may retire and
|
||||||
hard equality gates when that population is stable, and become named workload
|
shrink normally; any growth is a hard leak failure when the visible and
|
||||||
warnings when the authoritative population changed. Particles, emitters, and
|
authoritative workload is stable, and a named owner-growth warning when that
|
||||||
active scripts are always transient workload warnings rather than silent
|
workload changed. Particles, emitters, and active scripts are always transient
|
||||||
exclusions.
|
workload warnings rather than silent exclusions. The report therefore says
|
||||||
|
which canonical owner grew without mistaking legitimate retirement for a leak.
|
||||||
Managed used/committed deltas are reported diagnostically. Existing
|
Managed used/committed deltas are reported diagnostically. Existing
|
||||||
working/private-memory, update-p95, and allocation-p50 thresholds remain byte-
|
working/private-memory, update-p95, and allocation-p50 thresholds remain byte-
|
||||||
for-byte unchanged as the secondary residency/performance guard.
|
for-byte unchanged as the secondary residency/performance guard.
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,21 @@ public sealed class ConnectedR6SoakContractTests
|
||||||
.ToArray();
|
.ToArray();
|
||||||
|
|
||||||
Assert.Equal(ExpectedCheckpointNames, checkpoints);
|
Assert.Equal(ExpectedCheckpointNames, checkpoints);
|
||||||
|
|
||||||
|
string[] teleports = File.ReadAllLines(Path.Combine(
|
||||||
|
FindRepoRoot(),
|
||||||
|
"tools",
|
||||||
|
"connected-r6-soak.route.txt"))
|
||||||
|
.Select(static line => line.Trim())
|
||||||
|
.Where(static line => line.StartsWith(
|
||||||
|
"command /teleloc ",
|
||||||
|
StringComparison.Ordinal))
|
||||||
|
.ToArray();
|
||||||
|
Assert.Equal(9, teleports.Length);
|
||||||
|
Assert.All(teleports, static line => Assert.EndsWith(
|
||||||
|
" 1 0 0 0",
|
||||||
|
line,
|
||||||
|
StringComparison.Ordinal));
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|
@ -67,6 +82,14 @@ public sealed class ConnectedR6SoakContractTests
|
||||||
"missing render-world synchronization fields",
|
"missing render-world synchronization fields",
|
||||||
source,
|
source,
|
||||||
StringComparison.Ordinal);
|
StringComparison.Ordinal);
|
||||||
|
Assert.Contains(
|
||||||
|
"canonical cache '$field' grew",
|
||||||
|
source,
|
||||||
|
StringComparison.Ordinal);
|
||||||
|
Assert.Contains(
|
||||||
|
"if ($workloadChanged) { $warnings.Add($message) }",
|
||||||
|
source,
|
||||||
|
StringComparison.Ordinal);
|
||||||
|
|
||||||
string[] zeroFields =
|
string[] zeroFields =
|
||||||
[
|
[
|
||||||
|
|
|
||||||
|
|
@ -4,8 +4,9 @@
|
||||||
|
|
||||||
sleep 15000
|
sleep 15000
|
||||||
|
|
||||||
# Caul baseline + local movement/jump/combat exercise.
|
# Caul baseline + local movement/jump/combat exercise. Every teleloc carries
|
||||||
command /teleloc 0xC95B0001 14.8 0.3 12.005
|
# the identity quaternion so each timed turn begins from the same heading.
|
||||||
|
command /teleloc 0xC95B0001 14.8 0.3 12.005 1 0 0 0
|
||||||
wait materialized 1 60000
|
wait materialized 1 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
@ -43,7 +44,7 @@ sleep 7400
|
||||||
checkpoint caul-baseline
|
checkpoint caul-baseline
|
||||||
|
|
||||||
# Sawato baseline.
|
# Sawato baseline.
|
||||||
command /teleloc 0x3032001C 83.3 89.2 133.005
|
command /teleloc 0x3032001C 83.3 89.2 133.005 1 0 0 0
|
||||||
wait materialized 2 60000
|
wait materialized 2 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
@ -53,7 +54,7 @@ sleep 22000
|
||||||
checkpoint sawato-baseline
|
checkpoint sawato-baseline
|
||||||
|
|
||||||
# Rynthid world-edge streaming.
|
# Rynthid world-edge streaming.
|
||||||
command /teleloc 0xF6820033 145.7 49.855 58.010
|
command /teleloc 0xF6820033 145.7 49.855 58.010 1 0 0 0
|
||||||
wait materialized 3 60000
|
wait materialized 3 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
@ -63,7 +64,7 @@ sleep 22000
|
||||||
checkpoint rynthid
|
checkpoint rynthid
|
||||||
|
|
||||||
# Aerlinthe dense island.
|
# Aerlinthe dense island.
|
||||||
command /teleloc 0x09040008 11.4 188.6 87.705
|
command /teleloc 0x09040008 11.4 188.6 87.705 1 0 0 0
|
||||||
wait materialized 4 60000
|
wait materialized 4 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
@ -73,7 +74,7 @@ sleep 22000
|
||||||
checkpoint aerlinthe
|
checkpoint aerlinthe
|
||||||
|
|
||||||
# Same-location lifecycle revisit.
|
# Same-location lifecycle revisit.
|
||||||
command /teleloc 0x3032001C 83.3 89.2 133.005
|
command /teleloc 0x3032001C 83.3 89.2 133.005 1 0 0 0
|
||||||
wait materialized 5 60000
|
wait materialized 5 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
@ -83,7 +84,7 @@ sleep 22000
|
||||||
checkpoint sawato-return
|
checkpoint sawato-return
|
||||||
|
|
||||||
# Holtburg mixed-town workload + local exercise.
|
# Holtburg mixed-town workload + local exercise.
|
||||||
command /teleloc 0xA9B40019 84.0 7.1 94.005
|
command /teleloc 0xA9B40019 84.0 7.1 94.005 1 0 0 0
|
||||||
wait materialized 6 60000
|
wait materialized 6 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
@ -121,7 +122,7 @@ sleep 7400
|
||||||
checkpoint holtburg
|
checkpoint holtburg
|
||||||
|
|
||||||
# Caul return/leak oracle + final local exercise.
|
# Caul return/leak oracle + final local exercise.
|
||||||
command /teleloc 0xC95B0001 14.8 0.3 12.005
|
command /teleloc 0xC95B0001 14.8 0.3 12.005 1 0 0 0
|
||||||
wait materialized 7 60000
|
wait materialized 7 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
@ -162,7 +163,7 @@ checkpoint caul-return
|
||||||
# circuit intentionally warms bounded DAT/GPU caches, so comparing the cold
|
# circuit intentionally warms bounded DAT/GPU caches, so comparing the cold
|
||||||
# first Caul against its warm return would punish successful source-window
|
# first Caul against its warm return would punish successful source-window
|
||||||
# retirement. This circuit loads no new destination content.
|
# retirement. This circuit loads no new destination content.
|
||||||
command /teleloc 0x3032001C 83.3 89.2 133.005
|
command /teleloc 0x3032001C 83.3 89.2 133.005 1 0 0 0
|
||||||
wait materialized 8 60000
|
wait materialized 8 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
@ -171,7 +172,7 @@ input up MovementTurnRight
|
||||||
sleep 22000
|
sleep 22000
|
||||||
checkpoint sawato-plateau
|
checkpoint sawato-plateau
|
||||||
|
|
||||||
command /teleloc 0xC95B0001 14.8 0.3 12.005
|
command /teleloc 0xC95B0001 14.8 0.3 12.005 1 0 0 0
|
||||||
wait materialized 9 60000
|
wait materialized 9 60000
|
||||||
sleep 8000
|
sleep 8000
|
||||||
input down MovementTurnRight
|
input down MovementTurnRight
|
||||||
|
|
|
||||||
|
|
@ -386,34 +386,30 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
|
||||||
|
|
||||||
foreach ($field in @(
|
foreach ($field in @(
|
||||||
'loadedLandblocks',
|
'loadedLandblocks',
|
||||||
'visibleLandblocks',
|
'totalLandblocks')) {
|
||||||
'totalLandblocks',
|
|
||||||
'pendingLiveTeardowns',
|
|
||||||
'pendingLandblockRetirements',
|
|
||||||
'meshRenderData',
|
|
||||||
'meshAtlasArrays',
|
|
||||||
'meshEstimatedBytes',
|
|
||||||
'stagedMeshUploads',
|
|
||||||
'stagedMeshBytes',
|
|
||||||
'trackedGpuBytes',
|
|
||||||
'trackedGpuBuffers',
|
|
||||||
'trackedGpuTextures',
|
|
||||||
'ownedCompositeTextures',
|
|
||||||
'compositeTextureOwners',
|
|
||||||
'activeParticleTextures',
|
|
||||||
'particleTextureOwners',
|
|
||||||
'compositeWarmupPending')) {
|
|
||||||
$first = $caulReturn.resources.PSObject.Properties[$field]
|
$first = $caulReturn.resources.PSObject.Properties[$field]
|
||||||
$plateau = $caulPlateau.resources.PSObject.Properties[$field]
|
$plateau = $caulPlateau.resources.PSObject.Properties[$field]
|
||||||
if ($null -eq $first -or $null -eq $plateau) {
|
if ($null -eq $first -or $null -eq $plateau) {
|
||||||
$failures.Add("canonical Caul comparison is missing resources.$field")
|
$failures.Add("canonical Caul comparison is missing resources.$field")
|
||||||
}
|
}
|
||||||
elseif ([long]$first.Value -ne [long]$plateau.Value) {
|
elseif ([long]$first.Value -ne [long]$plateau.Value) {
|
||||||
$failures.Add("Caul plateau canonical owner '$field' changed $($first.Value) -> $($plateau.Value)")
|
$failures.Add("Caul plateau deterministic world domain '$field' changed $($first.Value) -> $($plateau.Value)")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
$populationChanged = $false
|
$workloadChanged = $false
|
||||||
|
$firstVisible =
|
||||||
|
$caulReturn.resources.PSObject.Properties['visibleLandblocks']
|
||||||
|
$plateauVisible =
|
||||||
|
$caulPlateau.resources.PSObject.Properties['visibleLandblocks']
|
||||||
|
if ($null -eq $firstVisible -or $null -eq $plateauVisible) {
|
||||||
|
$failures.Add('canonical Caul workload comparison is missing resources.visibleLandblocks')
|
||||||
|
}
|
||||||
|
elseif ([long]$firstVisible.Value -ne [long]$plateauVisible.Value) {
|
||||||
|
$workloadChanged = $true
|
||||||
|
$warnings.Add("Caul plateau visible workload changed $($firstVisible.Value) -> $($plateauVisible.Value)")
|
||||||
|
}
|
||||||
|
|
||||||
foreach ($field in @(
|
foreach ($field in @(
|
||||||
'worldEntities',
|
'worldEntities',
|
||||||
'animatedEntities',
|
'animatedEntities',
|
||||||
|
|
@ -425,11 +421,34 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
|
||||||
$failures.Add("canonical Caul population comparison is missing resources.$field")
|
$failures.Add("canonical Caul population comparison is missing resources.$field")
|
||||||
}
|
}
|
||||||
elseif ([long]$first.Value -ne [long]$plateau.Value) {
|
elseif ([long]$first.Value -ne [long]$plateau.Value) {
|
||||||
$populationChanged = $true
|
$workloadChanged = $true
|
||||||
$warnings.Add("Caul plateau authoritative population '$field' changed $($first.Value) -> $($plateau.Value)")
|
$warnings.Add("Caul plateau authoritative population '$field' changed $($first.Value) -> $($plateau.Value)")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
foreach ($field in @(
|
||||||
|
'meshRenderData',
|
||||||
|
'meshAtlasArrays',
|
||||||
|
'meshEstimatedBytes',
|
||||||
|
'trackedGpuBytes',
|
||||||
|
'trackedGpuBuffers',
|
||||||
|
'trackedGpuTextures',
|
||||||
|
'ownedCompositeTextures',
|
||||||
|
'compositeTextureOwners',
|
||||||
|
'activeParticleTextures',
|
||||||
|
'particleTextureOwners')) {
|
||||||
|
$first = $caulReturn.resources.PSObject.Properties[$field]
|
||||||
|
$plateau = $caulPlateau.resources.PSObject.Properties[$field]
|
||||||
|
if ($null -eq $first -or $null -eq $plateau) {
|
||||||
|
$failures.Add("canonical Caul cache comparison is missing resources.$field")
|
||||||
|
}
|
||||||
|
elseif ([long]$plateau.Value -gt [long]$first.Value) {
|
||||||
|
$message = "Caul plateau canonical cache '$field' grew $($first.Value) -> $($plateau.Value)"
|
||||||
|
if ($workloadChanged) { $warnings.Add($message) }
|
||||||
|
else { $failures.Add($message) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
foreach ($field in @(
|
foreach ($field in @(
|
||||||
'particleBindings',
|
'particleBindings',
|
||||||
'particleOwners',
|
'particleOwners',
|
||||||
|
|
@ -441,9 +460,9 @@ function Add-CanonicalCheckpointFailures([Diagnostics.Process]$Client) {
|
||||||
if ($null -eq $first -or $null -eq $plateau) {
|
if ($null -eq $first -or $null -eq $plateau) {
|
||||||
$failures.Add("canonical Caul owner comparison is missing resources.$field")
|
$failures.Add("canonical Caul owner comparison is missing resources.$field")
|
||||||
}
|
}
|
||||||
elseif ([long]$first.Value -ne [long]$plateau.Value) {
|
elseif ([long]$plateau.Value -gt [long]$first.Value) {
|
||||||
$message = "Caul plateau population-sensitive owner '$field' changed $($first.Value) -> $($plateau.Value)"
|
$message = "Caul plateau population-sensitive owner '$field' grew $($first.Value) -> $($plateau.Value)"
|
||||||
if ($populationChanged) { $warnings.Add($message) }
|
if ($workloadChanged) { $warnings.Add($message) }
|
||||||
else { $failures.Add($message) }
|
else { $failures.Add($message) }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue