From 86e0dc4655eeede35607c86ab04b3d16f35654a2 Mon Sep 17 00:00:00 2001 From: Erik Date: Sun, 5 Jul 2026 22:14:31 +0200 Subject: [PATCH] fix(pipeline): MP1b review - bake CLI determinism + bounded memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarially-verified review findings 1 and 10: (1) The bake pipeline no longer accumulates every decoded ObjectMeshData in one ConcurrentBag before writing (multi-GB OOM risk on the full bake), and no longer writes blobs in thread-completion order (which violated the plan's "bakes must be byte-reproducible run-to-run"). New shape in BakeRunner: build the FULL id list, sort by PakKey, chunk into 512-id batches; Parallel.ForEach WITHIN each batch; sort each batch's results by key and AddBlob sequentially; release the batch. Batches are contiguous key ranges, so the blob region lands in global key order regardless of thread scheduling, and peak memory is one batch's output. Side-staged particle-preload meshes drain per batch into a key-deduped map (first instance wins — per-id extraction output is deterministic, so instance choice cannot affect bytes) and are written after all batches, sorted by key, skipping keys already written. Program.cs is now a thin arg-parsing shell over the public BakeRunner so the new dat-gated byte-reproducibility test can drive the REAL pipeline: tests/AcDream.Bake.Tests (new project, rule 6; registered in slnx; no Silk.NET in its resolved dependency graph — verified) bakes the same 9-id mixed fixture twice with DIFFERENT thread counts (8 vs 3 — thread scheduling was the nondeterminism source) and asserts the two pak files are byte-identical. Ran for real against the dats on this machine: green. (10) The isSetup argument for EnvCell extraction now matches at both call sites (BakeRunner and PakEquivalenceTests both pass false) and is documented at each: the runtime's own request sites (WbMeshAdapter.IncrementRefCount / EnsureLoaded) pass isSetup: false for every MeshRef id including cell-geometry ids. The parameter is currently dead in MeshExtractor.PrepareMeshData (dispatch is on the resolved dat type), but two disagreeing call sites were a latent trap. Header FormatVersion is no longer set by the bake (PakWriter stamps it per review finding 2, previous commit). --- AcDream.slnx | 1 + src/AcDream.Bake/BakeRunner.cs | 248 ++++++++++++++++++ src/AcDream.Bake/Program.cs | 216 +-------------- .../AcDream.Bake.Tests.csproj | 29 ++ .../BakeDeterminismTests.cs | 79 ++++++ .../PakEquivalenceTests.cs | 6 +- 6 files changed, 371 insertions(+), 208 deletions(-) create mode 100644 src/AcDream.Bake/BakeRunner.cs create mode 100644 tests/AcDream.Bake.Tests/AcDream.Bake.Tests.csproj create mode 100644 tests/AcDream.Bake.Tests/BakeDeterminismTests.cs diff --git a/AcDream.slnx b/AcDream.slnx index c00c5246..abf3defb 100644 --- a/AcDream.slnx +++ b/AcDream.slnx @@ -16,6 +16,7 @@ + diff --git a/src/AcDream.Bake/BakeRunner.cs b/src/AcDream.Bake/BakeRunner.cs new file mode 100644 index 00000000..18db6c2b --- /dev/null +++ b/src/AcDream.Bake/BakeRunner.cs @@ -0,0 +1,248 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AcDream.Content; +using AcDream.Content.Pak; +using DatReaderWriter; +using DatReaderWriter.DBObjs; +using DatReaderWriter.Options; + +namespace AcDream.Bake; + +/// Options for one bake run (parsed from CLI args by Program, or built directly by tests). +public sealed record BakeOptions { + public required string DatDir { get; init; } + public required string OutPath { get; init; } + public HashSet? IdFilter { get; init; } + public HashSet? LandblockFilter { get; init; } + public int Threads { get; init; } = System.Environment.ProcessorCount; +} + +/// +/// The bake pipeline: enumerate ids -> sorted-batch parallel extraction -> +/// deterministic pak write. Public (rather than inline in Program) so the +/// dat-gated byte-reproducibility test can drive the REAL pipeline. +/// +/// Determinism + bounded memory (review finding 1): the full id +/// list is built first and sorted by pak key; work proceeds in fixed-size +/// batches — extraction is parallel WITHIN a batch, then the batch's results +/// are sorted by key and written sequentially before the next batch starts. +/// Batches are contiguous key ranges, so the blob region ends up in global +/// key order regardless of thread scheduling, and memory is bounded by one +/// batch's decoded output instead of the whole game's. Side-staged +/// particle-preload meshes are deduped into a keyed map as each batch +/// drains (first instance wins — extraction output per id is +/// deterministic, so instance choice cannot affect bytes) and written after +/// all batches, sorted by key, skipping keys already written. +/// +public static class BakeRunner { + /// Ids extracted in parallel per batch before the sequential sorted write. Bounds peak memory (~512 decoded ObjectMeshData) while keeping the workers busy. + private const int BatchSize = 512; + + public static int Run(BakeOptions options) { + Console.WriteLine("acdream-bake"); + Console.WriteLine($"dat dir: {options.DatDir}"); + Console.WriteLine($"out: {options.OutPath}"); + Console.WriteLine($"threads: {options.Threads}"); + Console.WriteLine(); + + var sw = Stopwatch.StartNew(); + using var dats = new DatCollection(options.DatDir, DatAccessType.Read); + using var datReaderWriter = new BakeDatCollectionAdapter(dats); + var extractorLogger = new ConsoleErrorLogger(nameof(MeshExtractor)); + + // Thread-safe side-stage sink for particle-preload meshes MeshExtractor + // emits mid-extraction (MP1a documented contract; extractor is shared + // across the batch workers). + var sideStaged = new ConcurrentQueue(); + var extractor = new MeshExtractor(datReaderWriter, extractorLogger, data => sideStaged.Enqueue(data)); + + // ---- enumeration ----------------------------------------------------- + + var gfxObjIds = dats.GetAllIdsOfType().ToList(); + var setupIds = dats.GetAllIdsOfType().ToList(); + var envCellIds = EnumerateEnvCellIds(dats, options.LandblockFilter); + + if (options.IdFilter is not null) { + gfxObjIds = gfxObjIds.Where(options.IdFilter.Contains).ToList(); + setupIds = setupIds.Where(options.IdFilter.Contains).ToList(); + envCellIds = envCellIds.Where(options.IdFilter.Contains).ToList(); + } + + // Full work list sorted by pak key — the batching below preserves this + // global order on disk (determinism precondition). + var work = new List<(ulong Key, PakAssetType Type, uint FileId)>(gfxObjIds.Count + setupIds.Count + envCellIds.Count); + work.AddRange(gfxObjIds.Select(id => (PakKey.Compose(PakAssetType.GfxObjMesh, id), PakAssetType.GfxObjMesh, id))); + work.AddRange(setupIds.Select(id => (PakKey.Compose(PakAssetType.SetupMesh, id), PakAssetType.SetupMesh, id))); + work.AddRange(envCellIds.Select(id => (PakKey.Compose(PakAssetType.EnvCellMesh, id), PakAssetType.EnvCellMesh, id))); + work.Sort((a, b) => a.Key.CompareTo(b.Key)); + + Console.WriteLine($"enumerated: {gfxObjIds.Count:N0} GfxObj, {setupIds.Count:N0} Setup, {envCellIds.Count:N0} EnvCell " + + $"({work.Count:N0} total)"); + Console.WriteLine(); + + // ---- sorted-batch extraction + streaming write ------------------------- + + var header = new PakHeader { + PortalIteration = (uint)dats.Portal.Iteration!.CurrentIteration, + CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration, + HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration, + LanguageIteration = (uint)dats.Local.Iteration!.CurrentIteration, + BakeToolVersion = 1, + }; + + var failures = new ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)>(); + var writtenKeys = new HashSet(); + var sideStagedByKey = new Dictionary(); + long completed = 0; + int writtenCounts0 = 0, writtenCounts1 = 0, writtenCounts2 = 0; // GfxObj, Setup, EnvCell + var lastProgressReport = Stopwatch.StartNew(); + + using (var writer = new PakWriter(options.OutPath, header)) { + for (int batchStart = 0; batchStart < work.Count; batchStart += BatchSize) { + var batch = work.Skip(batchStart).Take(BatchSize).ToList(); + var batchResults = new ConcurrentBag<(ulong Key, PakAssetType Type, ObjectMeshData Data)>(); + + Parallel.ForEach( + batch, + new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, + item => { + var (key, type, fileId) = item; + try { + // EnvCell entries store the cell's synthetic geometry mesh + // (bit 32 set — see MeshExtractor.PrepareMeshData's EnvCell + // branch); the cell's static objects are covered by their + // own GfxObj/Setup entries. + ulong extractorId = type == PakAssetType.EnvCellMesh ? fileId | 0x1_0000_0000UL : fileId; + // isSetup: matches the runtime's own request sites — + // WbMeshAdapter.IncrementRefCount/EnsureLoaded pass + // isSetup: false for every MeshRef id (incl. cell-geometry + // ids); only true Setup extraction passes true. (Review + // finding 10 — keep aligned with PakEquivalenceTests.) + bool isSetup = type == PakAssetType.SetupMesh; + var data = extractor.PrepareMeshData(extractorId, isSetup); + if (data is not null) { + batchResults.Add((key, type, data)); + } + else { + failures.Add((type, fileId, "extractor returned null (no polygons or unresolvable id)")); + } + } + catch (Exception ex) { + // A malformed dat entry skips that id — never fatal to the + // bake, matching the runtime's own per-id behavior. + failures.Add((type, fileId, ex.Message)); + } + + Interlocked.Increment(ref completed); + }); + + // Sequential sorted write: batch results in key order. + foreach (var (key, type, data) in batchResults.OrderBy(r => r.Key)) { + writer.AddBlob(key, data); + writtenKeys.Add(key); + switch (type) { + case PakAssetType.GfxObjMesh: writtenCounts0++; break; + case PakAssetType.SetupMesh: writtenCounts1++; break; + case PakAssetType.EnvCellMesh: writtenCounts2++; break; + } + } + + // Drain side-staged preloads per batch into the dedup map so the + // queue never grows past one batch's emissions (memory bound); + // written after all batches, sorted (see below). + while (sideStaged.TryDequeue(out var staged)) { + uint fileId = (uint)(staged.ObjectId & 0xFFFFFFFFu); + ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId); + if (!sideStagedByKey.ContainsKey(key)) sideStagedByKey[key] = staged; + } + + if (lastProgressReport.Elapsed.TotalSeconds >= 5 || batchStart + BatchSize >= work.Count) { + ReportProgress(completed, work.Count, failures.Count, sw.Elapsed); + lastProgressReport.Restart(); + } + } + + // Side-staged preload meshes not already covered by a primary entry: + // sorted by key for deterministic placement. + int sideStagedWritten = 0, sideStagedDuped = 0; + foreach (var (key, data) in sideStagedByKey.OrderBy(kv => kv.Key)) { + if (writtenKeys.Contains(key)) { sideStagedDuped++; continue; } + writer.AddBlob(key, data); + writtenKeys.Add(key); + sideStagedWritten++; + } + + writer.Finish(); + sw.Stop(); + + var outSize = new FileInfo(options.OutPath).Length; + Console.WriteLine(); + Console.WriteLine("=== bake summary ==="); + Console.WriteLine($" GfxObj baked: {writtenCounts0:N0}"); + Console.WriteLine($" Setup baked: {writtenCounts1:N0}"); + Console.WriteLine($" EnvCell baked: {writtenCounts2:N0}"); + Console.WriteLine($" side-staged baked (particle preload GfxObjs): {sideStagedWritten:N0} ({sideStagedDuped:N0} deduped)"); + Console.WriteLine($" total blobs: {writtenKeys.Count:N0}"); + Console.WriteLine($" failures: {failures.Count:N0}"); + Console.WriteLine($" elapsed: {sw.Elapsed.TotalSeconds:F1} s"); + Console.WriteLine($" output size: {outSize / 1024.0 / 1024.0:F1} MB"); + Console.WriteLine($" output path: {options.OutPath}"); + } + + if (!failures.IsEmpty) { + Console.WriteLine(); + Console.WriteLine($"failures ({failures.Count}):"); + foreach (var (type, fileId, reason) in failures.OrderBy(f => f.FileId).Take(200)) { + Console.WriteLine($" {type,-12} 0x{fileId:X8}: {reason}"); + } + if (failures.Count > 200) Console.WriteLine($" ... and {failures.Count - 200} more"); + } + + return 0; + } + + private static void ReportProgress(long done, int total, int failures, TimeSpan elapsed) { + double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0; + double etaSeconds = rate > 0 ? (total - done) / rate : 0; + Console.WriteLine($"[{elapsed:hh\\:mm\\:ss}] baked {done:N0}/{total:N0}, failures={failures:N0}, " + + $"elapsed={elapsed.TotalSeconds:F0}s, ETA={etaSeconds:F0}s"); + } + + /// + /// Enumerates EnvCell ids by walking the cell dat's LandBlockInfo entries + /// (0xFFFE low 16 bits) and, for each, the NumCells-derived cell id range — + /// GetAllIdsOfType<T>() does not cover cell-dat range-based types + /// (DatReaderWriter's documented limitation; see the low-16-bit bucketing + /// idiom in src/AcDream.Cli/Program.cs's CountCellByLow16, and the + /// firstCellId/NumCells hydration idiom in GameWindow.BuildPhysicsDatBundle + /// / BuildInteriorEntitiesForStreaming). + /// + private static List EnumerateEnvCellIds(DatCollection dats, HashSet? landblockFilter) { + var landblockInfoIds = new List(); + foreach (var file in dats.Cell.Tree) { + if ((file.Id & 0xFFFFu) != 0xFFFEu) continue; + uint landblockId = file.Id & 0xFFFF0000u; + if (landblockFilter is not null && !landblockFilter.Contains(landblockId)) continue; + landblockInfoIds.Add(file.Id); + } + + var envCellIds = new List(); + foreach (var lbInfoId in landblockInfoIds) { + if (!dats.Cell.TryGet(lbInfoId, out var lbInfo) || lbInfo is null) continue; + if (lbInfo.NumCells == 0) continue; + + uint landblockId = lbInfoId & 0xFFFF0000u; + uint firstCellId = landblockId | 0x0100u; + for (uint offset = 0; offset < lbInfo.NumCells; offset++) { + envCellIds.Add(firstCellId + offset); + } + } + return envCellIds; + } +} diff --git a/src/AcDream.Bake/Program.cs b/src/AcDream.Bake/Program.cs index c8288829..2d7ee8c0 100644 --- a/src/AcDream.Bake/Program.cs +++ b/src/AcDream.Bake/Program.cs @@ -1,27 +1,15 @@ using System; -using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; -using System.Threading; -using System.Threading.Tasks; using AcDream.Bake; -using AcDream.Content; -using AcDream.Content.Pak; -using DatReaderWriter; -using DatReaderWriter.DBObjs; -using DatReaderWriter.Options; -using Environment = System.Environment; // acdream-bake: offline CLI producing a versioned pak file containing every // ObjectMeshData the client's decode workers would otherwise produce at // runtime (GfxObj / Setup / EnvCell mesh+texture payloads). // -// Drives AcDream.Content.MeshExtractor — the SAME extraction code the live -// client runs — so pak-vs-live equivalence is by construction (verified by -// the Task 6 dat-gated equivalence suite), not by a second hand-written -// extraction path. +// Thin arg-parsing shell over BakeRunner (the real pipeline lives there so +// the dat-gated byte-reproducibility test can drive it directly). // // Plan: docs/superpowers/plans/2026-07-05-mp1b-pak-and-bake.md, Task 5. @@ -66,160 +54,13 @@ if (!Directory.Exists(datDir)) { outPath ??= Path.Combine(datDir, "acdream.pak"); -Console.WriteLine("acdream-bake"); -Console.WriteLine($"dat dir: {datDir}"); -Console.WriteLine($"out: {outPath}"); -Console.WriteLine($"threads: {threads}"); -Console.WriteLine(); - -var sw = Stopwatch.StartNew(); -using var dats = new DatCollection(datDir, DatAccessType.Read); -using var datReaderWriter = new BakeDatCollectionAdapter(dats); -var extractorLogger = new ConsoleErrorLogger(nameof(MeshExtractor)); - -// A ConcurrentQueue collector for particle-preload meshes MeshExtractor -// side-stages mid-extraction (CollectEmittersFromScript). MeshExtractor is -// documented thread-safe for up to MaxParallelLoads (4) concurrent workers -// sharing one instance and requires this sink argument explicitly — no -// default — so a bake tool can't silently forget it and drop preloads. -// The bake tool treats each side-staged item as its own GfxObjMesh entry, -// deduplicated by id the same way the enumerated id set is. -var sideStaged = new ConcurrentQueue(); -var extractor = new MeshExtractor(datReaderWriter, extractorLogger, data => sideStaged.Enqueue(data)); - -// ---- enumeration --------------------------------------------------------- - -var gfxObjIds = dats.GetAllIdsOfType().ToList(); -var setupIds = dats.GetAllIdsOfType().ToList(); -var envCellIds = EnumerateEnvCellIds(dats, landblockFilter); - -if (idFilter is not null) { - gfxObjIds = gfxObjIds.Where(idFilter.Contains).ToList(); - setupIds = setupIds.Where(idFilter.Contains).ToList(); - envCellIds = envCellIds.Where(idFilter.Contains).ToList(); -} - -var work = new List<(PakAssetType Type, uint FileId)>(gfxObjIds.Count + setupIds.Count + envCellIds.Count); -work.AddRange(gfxObjIds.Select(id => (PakAssetType.GfxObjMesh, id))); -work.AddRange(setupIds.Select(id => (PakAssetType.SetupMesh, id))); -work.AddRange(envCellIds.Select(id => (PakAssetType.EnvCellMesh, id))); - -Console.WriteLine($"enumerated: {gfxObjIds.Count:N0} GfxObj, {setupIds.Count:N0} Setup, {envCellIds.Count:N0} EnvCell " + - $"({work.Count:N0} total)"); -Console.WriteLine(); - -// ---- parallel extract ----------------------------------------------------- - -var results = new ConcurrentBag<(PakAssetType Type, uint FileId, ObjectMeshData Data)>(); -var failures = new ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)>(); -long completed = 0; -var progressLock = new object(); -var lastProgressReport = Stopwatch.StartNew(); - -Parallel.ForEach( - work, - new ParallelOptions { MaxDegreeOfParallelism = threads }, - item => { - var (type, fileId) = item; - try { - ulong extractorId = type == PakAssetType.EnvCellMesh ? fileId | 0x1_0000_0000UL : fileId; - bool isSetup = type == PakAssetType.SetupMesh; - var data = extractor.PrepareMeshData(extractorId, isSetup); - if (data is not null) { - // Extractor echoes back its own resolved ObjectId (which may - // differ in the high bits for EnvCell synthetic-geometry - // requests) — always key the pak entry by the ENUMERATED - // fileId so lookups at read time are predictable. - results.Add((type, fileId, data)); - } - else { - failures.Add((type, fileId, "extractor returned null (no polygons or unresolvable id)")); - } - } - catch (Exception ex) { - // A malformed dat entry skips that id — never fatal to the bake, - // matching the runtime's own per-id try/catch behavior. - failures.Add((type, fileId, ex.Message)); - } - - long done = Interlocked.Increment(ref completed); - lock (progressLock) { - if (lastProgressReport.Elapsed.TotalSeconds >= 5) { - ReportProgress(done, work.Count, failures.Count, sw.Elapsed); - lastProgressReport.Restart(); - } - } - }); - -ReportProgress(completed, work.Count, failures.Count, sw.Elapsed); -Console.WriteLine(); - -// ---- write pak ----------------------------------------------------------- - -var header = new PakHeader { - FormatVersion = 1, - PortalIteration = (uint)dats.Portal.Iteration!.CurrentIteration, - CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration, - HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration, - LanguageIteration = (uint)dats.Local.Iteration!.CurrentIteration, - BakeToolVersion = 1, -}; - -var seenSideStagedIds = new HashSet(); -int written = 0, sideStagedWritten = 0, sideStagedDuped = 0; -using (var writer = new PakWriter(outPath, header)) { - foreach (var (type, fileId, data) in results) { - writer.AddBlob(PakKey.Compose(type, fileId), data); - written++; - } - - // Side-staged particle-preload GfxObj meshes: dedup by id against BOTH - // each other and the primary GfxObj set already written (a particle's - // preload GfxObj can also be independently enumerated as a top-level - // GfxObj — MeshExtractor.PrepareMeshData(id, isSetup:false) is - // deterministic, so re-extracting would just waste time, but we must - // never write the SAME pak key twice). - var primaryGfxObjIds = new HashSet(gfxObjIds); - while (sideStaged.TryDequeue(out var staged)) { - uint fileId = (uint)(staged.ObjectId & 0xFFFFFFFFu); - if (primaryGfxObjIds.Contains(fileId) || !seenSideStagedIds.Add(staged.ObjectId)) { - sideStagedDuped++; - continue; - } - writer.AddBlob(PakKey.Compose(PakAssetType.GfxObjMesh, fileId), staged); - sideStagedWritten++; - } - - writer.Finish(); -} - -sw.Stop(); - -var outSize = new FileInfo(outPath).Length; - -Console.WriteLine("=== bake summary ==="); -Console.WriteLine($" GfxObj baked: {results.Count(r => r.Type == PakAssetType.GfxObjMesh):N0}"); -Console.WriteLine($" Setup baked: {results.Count(r => r.Type == PakAssetType.SetupMesh):N0}"); -Console.WriteLine($" EnvCell baked: {results.Count(r => r.Type == PakAssetType.EnvCellMesh):N0}"); -Console.WriteLine($" side-staged baked (particle preload GfxObjs): {sideStagedWritten:N0} ({sideStagedDuped:N0} deduped)"); -Console.WriteLine($" total blobs: {written + sideStagedWritten:N0}"); -Console.WriteLine($" failures: {failures.Count:N0}"); -Console.WriteLine($" elapsed: {sw.Elapsed.TotalSeconds:F1} s"); -Console.WriteLine($" output size: {outSize / 1024.0 / 1024.0:F1} MB"); -Console.WriteLine($" output path: {outPath}"); - -if (failures.Count > 0) { - Console.WriteLine(); - Console.WriteLine($"failures ({failures.Count}):"); - foreach (var (type, fileId, reason) in failures.OrderBy(f => f.FileId).Take(200)) { - Console.WriteLine($" {type,-12} 0x{fileId:X8}: {reason}"); - } - if (failures.Count > 200) Console.WriteLine($" ... and {failures.Count - 200} more"); -} - -return 0; - -// ---- helpers --------------------------------------------------------------- +return BakeRunner.Run(new BakeOptions { + DatDir = datDir, + OutPath = outPath, + IdFilter = idFilter, + LandblockFilter = landblockFilter, + Threads = threads, +}); static HashSet ParseHexList(string? raw) { var result = new HashSet(); @@ -235,42 +76,3 @@ static HashSet ParseHexList(string? raw) { } return result; } - -static void ReportProgress(long done, int total, int failures, TimeSpan elapsed) { - double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0; - double etaSeconds = rate > 0 ? (total - done) / rate : 0; - Console.WriteLine($"[{elapsed:hh\\:mm\\:ss}] baked {done:N0}/{total:N0}, failures={failures:N0}, " + - $"elapsed={elapsed.TotalSeconds:F0}s, ETA={etaSeconds:F0}s"); -} - -/// -/// Enumerates EnvCell ids by walking the cell dat's LandBlockInfo entries -/// (0xFFFE low 16 bits) and, for each, the NumCells-derived cell id range — -/// GetAllIdsOfType<T>() does not cover cell-dat range-based types -/// (DatReaderWriter's documented limitation; see the low-16-bit bucketing -/// idiom in src/AcDream.Cli/Program.cs's CountCellByLow16, and the -/// firstCellId/NumCells hydration idiom in GameWindow.BuildPhysicsDatBundle -/// / BuildInteriorEntitiesForStreaming). -/// -static List EnumerateEnvCellIds(DatCollection dats, HashSet? landblockFilter) { - var landblockInfoIds = new List(); - foreach (var file in dats.Cell.Tree) { - if ((file.Id & 0xFFFFu) != 0xFFFEu) continue; - uint landblockId = file.Id & 0xFFFF0000u; - if (landblockFilter is not null && !landblockFilter.Contains(landblockId)) continue; - landblockInfoIds.Add(file.Id); - } - - var envCellIds = new List(); - foreach (var lbInfoId in landblockInfoIds) { - if (!dats.Cell.TryGet(lbInfoId, out var lbInfo) || lbInfo is null) continue; - if (lbInfo.NumCells == 0) continue; - - uint landblockId = lbInfoId & 0xFFFF0000u; - uint firstCellId = landblockId | 0x0100u; - for (uint offset = 0; offset < lbInfo.NumCells; offset++) { - envCellIds.Add(firstCellId + offset); - } - } - return envCellIds; -} diff --git a/tests/AcDream.Bake.Tests/AcDream.Bake.Tests.csproj b/tests/AcDream.Bake.Tests/AcDream.Bake.Tests.csproj new file mode 100644 index 00000000..d67af0e8 --- /dev/null +++ b/tests/AcDream.Bake.Tests/AcDream.Bake.Tests.csproj @@ -0,0 +1,29 @@ + + + + net10.0 + enable + enable + false + latest + true + + + + + + + + + + + + + + + + + + + diff --git a/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs b/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs new file mode 100644 index 00000000..5902fd68 --- /dev/null +++ b/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs @@ -0,0 +1,79 @@ +using System; +using System.Collections.Generic; +using System.IO; +using AcDream.Bake; + +namespace AcDream.Bake.Tests; + +/// +/// Dat-gated byte-reproducibility gate for the bake pipeline (review finding +/// 1): the plan's "bakes must be byte-reproducible run-to-run" is a whole- +/// FILE property, so it must be tested through the REAL BakeRunner — the +/// serializer-level determinism tests in Content.Tests can't see thread- +/// completion-order effects in the blob region. Skips cleanly when the dats +/// are absent (CI), same convention as +/// tests/AcDream.Core.Tests/Conformance/DatConcurrencyStressTests. +/// +public sealed class BakeDeterminismTests : IDisposable { + private readonly List _tempFiles = new(); + + private string NewTempPakPath() { + var path = Path.Combine(Path.GetTempPath(), $"acdream-baketest-{Guid.NewGuid():N}.pak"); + _tempFiles.Add(path); + return path; + } + + public void Dispose() { + foreach (var f in _tempFiles) { + try { if (File.Exists(f)) File.Delete(f); } catch { /* best effort cleanup */ } + } + } + + private static string? ResolveDatDir() { + // Mirrors ConformanceDats.ResolveDatDir (env var, then the well-known + // Documents fallback); duplicated because test projects can't + // reference each other's helpers. + var fromEnv = Environment.GetEnvironmentVariable("ACDREAM_DAT_DIR"); + if (!string.IsNullOrWhiteSpace(fromEnv) && Directory.Exists(fromEnv)) return fromEnv; + + var def = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + "Documents", "Asheron's Call"); + return Directory.Exists(def) ? def : null; + } + + [Fact] + public void Bake_SameIdSet_DifferentThreadCounts_ByteIdenticalPaks() { + var datDir = ResolveDatDir(); + if (datDir is null) return; // dats absent (CI) — skip, matching suite convention + + // Small mixed fixture: GfxObjs (incl. the #119 tricky ids), Setups + // (door setup carries emitters -> exercises the side-staged preload + // path), and a Holtburg EnvCell. Big enough to span multiple asset + // types; small enough to run in seconds. + var ids = new HashSet { + 0x01000001u, 0x010002B4u, 0x010008A8u, 0x010014C3u, + 0x02000001u, 0x020019FFu, 0x020005D8u, + 0xA9B40100u, 0xA9B40101u, + }; + + var pathA = NewTempPakPath(); + var pathB = NewTempPakPath(); + + // DIFFERENT thread counts on purpose: thread-completion order was the + // nondeterminism source the sorted-batching fix removes — identical + // bytes must hold regardless of parallelism. + int rcA = BakeRunner.Run(new BakeOptions { DatDir = datDir, OutPath = pathA, IdFilter = ids, Threads = 8 }); + int rcB = BakeRunner.Run(new BakeOptions { DatDir = datDir, OutPath = pathB, IdFilter = ids, Threads = 3 }); + + Assert.Equal(0, rcA); + Assert.Equal(0, rcB); + + var bytesA = File.ReadAllBytes(pathA); + var bytesB = File.ReadAllBytes(pathB); + Assert.True(bytesA.Length == bytesB.Length, + $"pak sizes differ between runs: {bytesA.Length} vs {bytesB.Length} bytes"); + Assert.True(bytesA.AsSpan().SequenceEqual(bytesB), + "two bakes of the same id set produced different bytes — the sorted-batching determinism guarantee is broken"); + } +} diff --git a/tests/AcDream.Content.Tests/PakEquivalenceTests.cs b/tests/AcDream.Content.Tests/PakEquivalenceTests.cs index a9de7286..707b4adb 100644 --- a/tests/AcDream.Content.Tests/PakEquivalenceTests.cs +++ b/tests/AcDream.Content.Tests/PakEquivalenceTests.cs @@ -59,7 +59,11 @@ public sealed class PakEquivalenceTests { var work = new List<(PakAssetType Type, uint FileId, ulong ExtractorId, bool IsSetup)>(); work.AddRange(gfxObjIds.Select(id => (PakAssetType.GfxObjMesh, id, (ulong)id, false))); work.AddRange(setupIds.Select(id => (PakAssetType.SetupMesh, id, (ulong)id, true))); - work.AddRange(envCellIds.Select(id => (PakAssetType.EnvCellMesh, id, id | 0x1_0000_0000UL, true))); + // isSetup: false for EnvCell — matches the runtime's own request sites + // (WbMeshAdapter.IncrementRefCount/EnsureLoaded pass isSetup: false for + // every MeshRef id, incl. cell-geometry ids) and BakeRunner's call site. + // (Review finding 10 — the two call sites previously disagreed.) + work.AddRange(envCellIds.Select(id => (PakAssetType.EnvCellMesh, id, id | 0x1_0000_0000UL, false))); // ---- LIVE extraction (golden) ---- var golden = new Dictionary<(PakAssetType, uint), ObjectMeshData>();