diff --git a/tools/connected-backend-differential.route.txt b/tools/connected-backend-differential.route.txt new file mode 100644 index 00000000..dfb2e6ba --- /dev/null +++ b/tools/connected-backend-differential.route.txt @@ -0,0 +1,77 @@ +# Campaign V slice V7: the connected route the GL-versus-Vulkan differential +# runs TWICE, once per backend, pairing the PNGs by name. +# +# Two separate launches have to agree pixel-for-pixel, which is a much stricter +# requirement than either connected route was written for. Everything below is +# in service of that: +# +# * Every teleloc carries the identity quaternion, so the heading at each stop +# is the same on both runs rather than whatever the previous heading was. +# * The world clock is pinned by the client-only time-of-day override +# (AcdreamCycleTimeOfDay cycles live -> 0.00 -> 0.25 -> 0.50, so three +# presses land on noon). Without it the sky gradient, the sun angle and +# every lit surface drift with wall time between the two launches, which is +# the same effect that makes the offline gate mask its top 280 rows. +# * Nothing moves the character. The only thing between arrival and capture is +# a settle window, because a timed turn cannot be relied on to stop at the +# same angle twice. +# * There is no `checkpoint` verb here. The differential's verdict is the +# paired PNG, and the ownership ledger has its own gate +# (run-connected-world-lifecycle-gate.ps1) that this route does not duplicate. +# +# MSAA is forced OFF by the gate script rather than here, on BOTH launches. +# Sample positions are not specified across implementations and cost 8.8% of the +# frame at 4x -- measured, plan section 5.5.16 -- so a differential run with MSAA +# on measures the two drivers' sample patterns and nothing else. +# +# Residual divergence this route cannot remove, and which is a finding rather +# than a failure: PHASE. Two launches cannot agree on an emitter's age, a +# creature's animation cursor, a portal's scrolling texture, or the cloud +# sheet's position -- the cloud sheet in particular advances with WALL time and +# not with the Dereth clock the override above pins, which is why the offline +# pixel gate masks its own top 280 rows. The V6m smoke run measured the shape +# of it at stop 1 with MSAA off: 18.52% of the frame differed, 89% of that in +# the top 240 rows (cloud sheet, treeline, the portal), and rows 300-720 -- +# terrain, blending, roads, buildings, character, entire retained UI -- +# contributed about 9,500 pixels between them, almost all of it thin outlines +# on silhouette edges. That is why every stop below is chosen for its static +# content and why the settle windows are long. + +wait world-ready 90000 +wait world-visible 30000 +sleep 8000 +input press AcdreamCycleTimeOfDay +input press AcdreamCycleTimeOfDay +input press AcdreamCycleTimeOfDay +sleep 5000 + +# 1. Holtburg: the dense outdoor town. Terrain, terrain blending, the road +# overlay, static world meshes, scenery and the whole retained UI in one +# frame. This is the SMOKE stop -- the one pair -Smoke compares on its own. +command /teleloc 0xA9B40019 84.0 7.1 94.005 1 0 0 0 +wait materialized 1 90000 +wait world-visible 30000 +sleep 25000 +screenshot holtburg_town 15000 + +# 2. Facility Hub: an interior EnvCell, and the durable fix for the campaign's +# oldest coverage gap. Plan section 5.1 records that the offline gate sees no +# EnvCell at all and that EnvCellRenderer's Vulkan arm is proven by one +# eyes-on frame; this stop is the first automated instrument that looks at +# one. Cell 0x164 is >= 0x100, which is what makes it indoor: walls, floor, +# per-cell lighting and the doorway clip region rather than terrain. +command /teleloc 0x8A020164 70.35 -40.66 -5.9 1 0 0 0 +wait materialized 2 90000 +wait world-visible 30000 +sleep 25000 +screenshot facility_hub_interior 15000 + +# 3. Aerlinthe: dense island geometry, the water edge and a long view. The +# stop that exercises fog and the far end of the depth range, where a +# precision divergence between the two backends would show first. +command /teleloc 0x09040008 11.4 188.6 87.705 1 0 0 0 +wait materialized 3 90000 +wait world-visible 30000 +sleep 25000 +screenshot aerlinthe_island 15000 +sleep 3000 diff --git a/tools/run-backend-differential-gate.ps1 b/tools/run-backend-differential-gate.ps1 new file mode 100644 index 00000000..00a3682d --- /dev/null +++ b/tools/run-backend-differential-gate.ps1 @@ -0,0 +1,380 @@ +<# +.SYNOPSIS + Campaign V slice V7 instrument: the GL-versus-Vulkan connected differential. + +.DESCRIPTION + Runs the SAME connected route twice against the local ACE server -- once with + ACDREAM_RENDER_BACKEND=gl, once with =vulkan -- pairs the screenshots by name, + compares each pair with the `compare-screenshots` CLI at the plan's pinned + tolerance, and prints one verdict table. + + This is the instrument for plan section 2's pixel-acceptance contract: + + compare-screenshots expected.png actual.png out.json + channel tolerance 2, maximum differing fraction 0.001, + MSAA off, ACDREAM_DAY_GROUP pinned, + at every deterministic checkpoint of the connected route. + + THREE THINGS ARE FORCED, and each is load-bearing. + + 1. ACDREAM_MSAA_SAMPLES=0 ON BOTH LAUNCHES. Multisample resolve positions + are explicitly unspecified across implementations. Plan section 5.5.16 + MEASURED what leaving them on costs: 8.83% of the frame differed at 4x, + 81,359 pixels of 921,600, essentially all of it hugging foliage and + silhouette edges. That is two drivers' sample patterns, not a renderer + divergence, and it swamps the 0.001 threshold by ninety-fold. Forcing it + off on both runs is what makes the remaining difference attributable. + Note that it must be forced on BOTH -- matching a GL run at 4x against a + Vulkan run at 0 would be worse than either. + + 2. ACDREAM_DAY_GROUP pinned. The sky animates and the Dereth clock advances + with wall time; two launches minutes apart cannot agree about either. + The route additionally pins the client-only time-of-day override. + + 3. A DESKTOP WITNESS per run, with the guards the repeat-run gate + (run-repeat-connected-gate.ps1) learned the hard way. The client's own + capture is the differential's subject, but a blank frame reads back as a + valid PNG (plan section 5.5.2), so a second instrument that shares + nothing with the renderer decides whether the run is worth comparing at + all. Its failure modes are the same three that gate documents: + + * an overlapping window or a locked screen -- CopyFromScreen grabs + whatever is composited at those coordinates, so run this with + nothing else on the primary monitor and the desktop unlocked; + * stray input -- a foregrounded window takes keystrokes and scrolls, + which moves the camera off the pinned stop and produces an enormous + spurious difference. The client log is scanned for camera-affecting + input afterwards and the run is aborted if any arrived; + * a hard kill -- ACE holds the session about three minutes and the + SECOND launch of this gate is the one that pays for it, so both runs + close through WM_CLOSE. + +.PARAMETER Out + Directory for both captures, the comparison reports and the verdict JSON. + +.PARAMETER Route + The route both backends run. Defaults to the differential route, which is + written for two-launch determinism -- see its header before substituting one. + +.PARAMETER Smoke + Compare only the FIRST stop. The route is truncated after its first + `screenshot` verb, so the run is short and the artifact records exactly what + ran. Use this to measure the distance between the backends without paying + for the whole route; the full route is the V7 gate. + +.PARAMETER Tolerance + Per-channel absolute difference allowed. Default 2, the plan's pinned value. + +.PARAMETER MaxDifferentFraction + Fraction of differing pixels allowed. Default 0.001, likewise pinned. + +.PARAMETER DayGroup + Sky day-group index pinned on both launches. Default 0. + +.PARAMETER MaskTopPixels + Height in pixels of a top band excluded from every comparison. DEFAULT 0 -- + nothing is masked, and the gate is a strict identity check on the whole + frame. + + It exists because the V6m smoke run measured what the band costs and the + number belongs where the next reader will find it. With the world clock + pinned by the route and MSAA off, a GL-versus-Vulkan pair at Holtburg + differed in 18.52% of the frame -- and 89% of that difference sat in the top + 240 rows, which is the scrolling cloud sheet plus the treeline it sits + behind. The cloud sheet advances with WALL time, not the Dereth clock, so + two launches minutes apart can never agree there; this is the same + population the offline pixel gate masks its top 280 rows for. Rows 300-720 + -- terrain, blending, roads, buildings, the character and the whole retained + UI -- contributed about 9,500 differing pixels between them. + + Whether V7 accepts a mask, pins the cloud phase, or fixes something the + smoke was actually seeing is V7's decision, not this script's, which is why + the default is off. Do NOT reach for this to make a failing gate pass: plan + section 7.1 rule 2. + +.PARAMETER MinRenderedBytes + Desktop-grab PNG size below which a run is called blank and the comparison + is refused. Default 500000, the repeat gate's measured separator. + +.PARAMETER CooldownSeconds + Wait between the two launches so ACE clears the first graceful logout. + Default 15. + +.EXAMPLE + tools/run-backend-differential-gate.ps1 -Out artifacts/v7-differential -Smoke + tools/run-backend-differential-gate.ps1 -Out artifacts/v7-differential +#> +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$Out, + [string]$Route, + [switch]$Smoke, + [int]$Tolerance = 2, + [double]$MaxDifferentFraction = 0.001, + [int]$DayGroup = 0, + [int]$MaskTopPixels = 0, + [int]$MinRenderedBytes = 500000, + [int]$CooldownSeconds = 15, + [int]$RouteTimeoutSeconds = 600, + [switch]$SkipBuild +) + +$ErrorActionPreference = 'Stop' +$repo = Split-Path -Parent $PSScriptRoot +$exe = Join-Path $repo 'src\AcDream.App\bin\Release\net10.0\AcDream.App.exe' +$cli = Join-Path $repo 'src\AcDream.Cli\bin\Release\net10.0\AcDream.Cli.dll' +if (-not $Route) { $Route = Join-Path $repo 'tools\connected-backend-differential.route.txt' } + +function Write-Step($message) { Write-Host "[differential] $message" } + +# --- Preconditions ------------------------------------------------------------ +if (Get-Process -Name AcDream.App -ErrorAction SilentlyContinue) { + throw 'AcDream.App is already running. This gate uses the shared test account and must not steal its session.' +} +if (@(Get-NetUDPEndpoint -LocalPort 9000 -ErrorAction SilentlyContinue).Count -eq 0) { + throw 'local ACE is not listening on UDP port 9000.' +} +if (-not (Test-Path $Route)) { throw "Route not found: $Route" } + +if (-not $SkipBuild) { + Write-Step 'building Release' + & dotnet build (Join-Path $repo 'AcDream.slnx') -c Release --nologo -v q | Out-Null + if ($LASTEXITCODE -ne 0) { throw "Release build failed with exit code $LASTEXITCODE." } +} +if (-not (Test-Path $exe)) { throw "Client not found at $exe." } +if (-not (Test-Path $cli)) { throw "AcDream.Cli not found at $cli." } + +if (Test-Path $Out) { Remove-Item -Recurse -Force $Out } +New-Item -ItemType Directory -Force -Path $Out | Out-Null + +# The route both runs execute. -Smoke truncates after the first capture; the +# truncated file is written into the artifact directory so what ran is on record. +$routeLines = Get-Content -LiteralPath $Route +if ($Smoke) { + $firstShot = -1 + for ($i = 0; $i -lt $routeLines.Count; $i++) { + if ($routeLines[$i].TrimStart().StartsWith('screenshot ')) { $firstShot = $i; break } + } + if ($firstShot -lt 0) { throw "The route has no screenshot verb to truncate at: $Route" } + $routeLines = @($routeLines[0..$firstShot]) + @('sleep 3000') + Write-Step "smoke mode: route truncated after its first capture ($($firstShot + 1) lines)" +} +$effectiveRoute = Join-Path $Out 'route.txt' +Set-Content -Encoding utf8 -Path $effectiveRoute -Value $routeLines + +Add-Type -AssemblyName System.Drawing +Add-Type @' +using System; +using System.Runtime.InteropServices; +public static class DifferentialGateSurface { + [DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd); + [DllImport("user32.dll")] public static extern bool ShowWindow(IntPtr hWnd, int nCmdShow); + [DllImport("user32.dll")] public static extern bool GetClientRect(IntPtr hWnd, out RECT r); + [DllImport("user32.dll")] public static extern bool ClientToScreen(IntPtr hWnd, ref POINT p); + [StructLayout(LayoutKind.Sequential)] public struct RECT { public int L, T, R, B; } + [StructLayout(LayoutKind.Sequential)] public struct POINT { public int X, Y; } +} +'@ + +function Invoke-BackendRun([string]$Backend) { + $dir = Join-Path $Out $Backend + New-Item -ItemType Directory -Force -Path $dir | Out-Null + $log = Join-Path $dir 'client.log' + $grab = Join-Path $dir 'desktop-grab.png' + + $env:ACDREAM_DAT_DIR = Join-Path $env:USERPROFILE "Documents\Asheron's Call" + $env:ACDREAM_LIVE = '1' + $env:ACDREAM_TEST_HOST = '127.0.0.1' + $env:ACDREAM_TEST_PORT = '9000' + $env:ACDREAM_TEST_USER = 'testaccount' + $env:ACDREAM_TEST_PASS = 'testpassword' + $env:ACDREAM_RETAIL_UI = '1' + $env:ACDREAM_DEVTOOLS = '0' + $env:ACDREAM_DAY_GROUP = "$DayGroup" + # Both launches, always. See the .DESCRIPTION note 1. + $env:ACDREAM_MSAA_SAMPLES = '0' + $env:ACDREAM_RENDER_BACKEND = $Backend + $env:ACDREAM_UI_PROBE_SCRIPT = $effectiveRoute + $env:ACDREAM_AUTOMATION_ARTIFACT_DIR = $dir + + Write-Step "launching $Backend (MSAA off, day group $DayGroup)" + # Normal window style: the desktop witness needs a composited frame. + $proc = Start-Process -FilePath $exe -WorkingDirectory $repo ` + -RedirectStandardOutput $log -RedirectStandardError "$log.err" -PassThru + + $completed = $false + $deadline = (Get-Date).AddSeconds($RouteTimeoutSeconds) + while ((Get-Date) -lt $deadline) { + if (Select-String -Path $log -SimpleMatch '[UI-PROBE] UI probe script complete' ` + -ErrorAction SilentlyContinue) { $completed = $true; break } + if ($proc.HasExited) { break } + Start-Sleep -Milliseconds 1000 + } + + # The desktop witness, taken while the client is still in its trailing sleep + # and therefore still rendering the last scene of the route. + $grabBytes = 0 + if (-not $proc.HasExited) { + try { + $proc.Refresh() + $h = $proc.MainWindowHandle + if ($h -ne [IntPtr]::Zero) { + [DifferentialGateSurface]::ShowWindow($h, 9) | Out-Null # SW_RESTORE + [DifferentialGateSurface]::SetForegroundWindow($h) | Out-Null + Start-Sleep -Milliseconds 1200 + $r = New-Object DifferentialGateSurface+RECT + [DifferentialGateSurface]::GetClientRect($h, [ref]$r) | Out-Null + $p = New-Object DifferentialGateSurface+POINT + [DifferentialGateSurface]::ClientToScreen($h, [ref]$p) | Out-Null + $w = $r.R - $r.L; $ht = $r.B - $r.T + if ($w -gt 0 -and $ht -gt 0) { + $bmp = New-Object System.Drawing.Bitmap $w, $ht + $g = [System.Drawing.Graphics]::FromImage($bmp) + $g.CopyFromScreen($p.X, $p.Y, 0, 0, (New-Object System.Drawing.Size $w, $ht)) + $bmp.Save($grab, [System.Drawing.Imaging.ImageFormat]::Png) + $g.Dispose(); $bmp.Dispose() + $grabBytes = (Get-Item $grab).Length + } + } + } + catch { Write-Step "desktop grab failed on ${Backend}: $_" } + } + + # Graceful close, always: a hard kill leaves ACE holding the session about + # three minutes, and the second launch is the one that would pay for it. + $app = Get-Process -Name AcDream.App -ErrorAction SilentlyContinue + if ($app) { + $app.CloseMainWindow() | Out-Null + if (-not $app.WaitForExit(20000)) { + Write-Step "WM_CLOSE timed out on $Backend; forcing" + $app | Stop-Process -Force + } + } + + Remove-Item Env:\ACDREAM_RENDER_BACKEND -ErrorAction SilentlyContinue + Remove-Item Env:\ACDREAM_MSAA_SAMPLES -ErrorAction SilentlyContinue + + $strayInput = @(Select-String -Path $log -Pattern 'ScrollUp|ScrollDown|ZoomIn|ZoomOut|CameraZoom' ` + -CaseSensitive -ErrorAction SilentlyContinue) + $witness = if ($grabBytes -ge $MinRenderedBytes) { 'RENDERED' } + elseif ($grabBytes -gt 0) { 'BLANK' } + else { 'NO-CAPTURE' } + + return [pscustomobject][ordered]@{ + Backend = $Backend + RouteCompleted = $completed + GrabBytes = $grabBytes + Witness = $witness + StrayInputLines = @($strayInput | Select-Object -First 5 | ForEach-Object { $_.Line }) + Screenshots = Join-Path $dir 'screenshots' + Log = $log + } +} + +$gl = Invoke-BackendRun 'gl' +Write-Step "cooldown ${CooldownSeconds}s so ACE clears the graceful logout" +Start-Sleep -Seconds $CooldownSeconds +$vk = Invoke-BackendRun 'vulkan' + +# --- Sanity, before any pixel is compared ------------------------------------ +$aborts = @() +foreach ($run in @($gl, $vk)) { + if (-not $run.RouteCompleted) { $aborts += "$($run.Backend): the route did not complete" } + if ($run.Witness -ne 'RENDERED') { + $aborts += "$($run.Backend): desktop witness says $($run.Witness) ($($run.GrabBytes) B) -- locked screen, an overlapping window, or a blank frame" + } + if ($run.StrayInputLines.Count -gt 0) { + $aborts += "$($run.Backend): camera-affecting input reached the window; the capture is not comparable" + } +} + +$pairs = @() +if ($aborts.Count -eq 0) { + $glShots = @(Get-ChildItem $gl.Screenshots -Filter *.png -ErrorAction SilentlyContinue) + if ($glShots.Count -eq 0) { $aborts += 'gl: no screenshots were captured' } + + # The comparer skips any pixel whose mask alpha is non-zero, so the excluded + # band is opaque and everything compared is transparent. Same construction as + # the offline pixel gate's sky mask; off unless -MaskTopPixels was given. + $maskPath = $null + if ($MaskTopPixels -gt 0 -and $glShots.Count -gt 0) { + $probeImage = [System.Drawing.Bitmap]::FromFile($glShots[0].FullName) + $mw = $probeImage.Width; $mh = $probeImage.Height + $probeImage.Dispose() + $mask = New-Object System.Drawing.Bitmap($mw, $mh, [System.Drawing.Imaging.PixelFormat]::Format32bppArgb) + $graphics = [System.Drawing.Graphics]::FromImage($mask) + $graphics.Clear([System.Drawing.Color]::FromArgb(0, 0, 0, 0)) + $opaque = New-Object System.Drawing.SolidBrush ([System.Drawing.Color]::FromArgb(255, 255, 0, 255)) + $graphics.FillRectangle($opaque, 0, 0, $mw, [Math]::Min($MaskTopPixels, $mh)) + $graphics.Dispose(); $opaque.Dispose() + $maskPath = Join-Path $Out 'mask.png' + $mask.Save($maskPath, [System.Drawing.Imaging.ImageFormat]::Png) + $mask.Dispose() + Write-Step "masking the top $MaskTopPixels rows of every pair" + } + + foreach ($shot in $glShots) { + $counterpart = Join-Path $vk.Screenshots $shot.Name + if (-not (Test-Path $counterpart)) { + $pairs += [pscustomobject][ordered]@{ + Stop = $shot.BaseName; Verdict = 'UNPAIRED' + DifferentPixels = $null; Fraction = $null; MaxChannelDelta = $null + } + continue + } + $report = Join-Path $Out "compare-$($shot.BaseName).json" + if ($maskPath) { + & dotnet $cli compare-screenshots $shot.FullName $counterpart $report $Tolerance $MaxDifferentFraction $maskPath | Out-Null + } + else { + & dotnet $cli compare-screenshots $shot.FullName $counterpart $report $Tolerance $MaxDifferentFraction | Out-Null + } + $verdict = Get-Content $report -Raw | ConvertFrom-Json + $outcome = if ($verdict.passed) { 'MATCH' } else { 'DIVERGED' } + $pairs += [pscustomobject][ordered]@{ + Stop = $shot.BaseName + Verdict = $outcome + DifferentPixels = $verdict.differentPixels + Fraction = $verdict.differentPixelFraction + MaxChannelDelta = $verdict.maximumChannelDelta + } + } +} + +# --- Verdict ------------------------------------------------------------------ +Write-Host '' +if ($aborts.Count -gt 0) { + Write-Host '[differential] ABORTED before comparison:' -ForegroundColor Red + $aborts | ForEach-Object { Write-Host " - $_" -ForegroundColor Red } +} +else { + $pairs | Format-Table -AutoSize | Out-String | Write-Host +} + +$diverged = @($pairs | Where-Object Verdict -ne 'MATCH') +$report = [pscustomobject][ordered]@{ + Passed = ($aborts.Count -eq 0 -and $diverged.Count -eq 0) + Smoke = [bool]$Smoke + Commit = (& git -C $repo rev-parse HEAD).Trim() + Route = $Route + EffectiveRoute = $effectiveRoute + ChannelTolerance = $Tolerance + MaxDifferentPixelFraction = $MaxDifferentFraction + MsaaSamples = 0 + DayGroup = $DayGroup + MaskTopPixels = $MaskTopPixels + Runs = @($gl, $vk) + Pairs = @($pairs) + Aborts = @($aborts) +} +$reportPath = Join-Path $Out 'differential.json' +$report | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath $reportPath -Encoding utf8 +Write-Step "report: $reportPath" + +if ($aborts.Count -gt 0) { exit 2 } +if ($diverged.Count -gt 0) { + Write-Host ("[differential] FAILED: {0}/{1} stop(s) diverged." -f $diverged.Count, $pairs.Count) -ForegroundColor Red + exit 1 +} +Write-Step ("PASS: {0}/{0} stop(s) match at tolerance {1} / fraction {2}." -f $pairs.Count, $Tolerance, $MaxDifferentFraction) +exit 0