diff --git a/src/AcDream.Bake/BakeRunner.cs b/src/AcDream.Bake/BakeRunner.cs index 9e5676d3..73ee1a33 100644 --- a/src/AcDream.Bake/BakeRunner.cs +++ b/src/AcDream.Bake/BakeRunner.cs @@ -1,98 +1,148 @@ -using System; using System.Collections.Concurrent; -using System.Collections.Generic; using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; +using System.Numerics; using AcDream.Content; using AcDream.Content.Pak; using DatReaderWriter; using DatReaderWriter.DBObjs; using DatReaderWriter.Options; +using DatEnvironment = DatReaderWriter.DBObjs.Environment; namespace AcDream.Bake; /// Options for one bake run (parsed from CLI args by Program, or built directly by tests). -public sealed record BakeOptions { +public sealed record BakeOptions +{ public required string DatDir { get; init; } public required string OutPath { get; init; } public HashSet? IdFilter { get; init; } public HashSet? LandblockFilter { get; init; } public int Threads { get; init; } = System.Environment.ProcessorCount; + public CancellationToken CancellationToken { get; init; } +} + +/// Compact result used by the full-scale gate and deterministic tests. +public sealed record BakeReport +{ + public required int GfxObjKeys { get; init; } + public required int SetupKeys { get; init; } + public required int EnvCellKeys { get; init; } + public required int UniqueEnvCellGeometries { get; init; } + public required int EnvCellAliases { get; init; } + public required int SideStagedKeys { get; init; } + public required int PhysicalBlobs { get; init; } + public required int TotalKeys { get; init; } + public required int Failures { get; init; } + public required TimeSpan Elapsed { get; init; } + public required long OutputBytes { get; init; } + public required long PeakWorkingSetBytes { get; init; } + + public double EnvCellDedupRatio => + UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries; } /// -/// The bake pipeline: enumerate ids -> sorted-batch parallel extraction -> -/// deterministic pak write. Public (rather than inline in Program) so the -/// dat-gated byte-reproducibility test can drive the REAL pipeline. -/// -/// Determinism + bounded memory (review finding 1): the full id -/// list is built first and sorted by pak key; work proceeds in fixed-size -/// batches — extraction is parallel WITHIN a batch, then the batch's results -/// are sorted by key and written sequentially before the next batch starts. -/// Batches are contiguous key ranges, so the blob region ends up in global -/// key order regardless of thread scheduling, and memory is bounded by one -/// batch's decoded output instead of the whole game's. Side-staged -/// particle-preload meshes are deduped into a keyed map as each batch -/// drains (first instance wins — extraction output per id is -/// deterministic, so instance choice cannot affect bytes) and written after -/// all batches, sorted by key, skipping keys already written. +/// Offline content pipeline: enumerate source IDs, catalog EnvCell geometry by +/// the same content identity used at runtime, extract each unique payload once, +/// and stream deterministic blobs plus ordinary TOC aliases. /// -public static class BakeRunner { - /// Ids extracted in parallel per batch before the sequential sorted write. Bounds peak memory (~512 decoded ObjectMeshData) while keeping the workers busy. +public static class BakeRunner +{ + /// + /// Items extracted in parallel per batch before sequential sorted write. + /// Bounds peak decoded output while keeping workers busy. + /// private const int BatchSize = 512; - public static int Run(BakeOptions options) { + public static int Run(BakeOptions options) + { + RunDetailed(options); + return 0; + } + + public static BakeReport RunDetailed(BakeOptions options) + { + ArgumentNullException.ThrowIfNull(options); + if (options.Threads <= 0) + throw new ArgumentOutOfRangeException(nameof(options), "thread count must be positive"); + + var cancellationToken = options.CancellationToken; + cancellationToken.ThrowIfCancellationRequested(); + Console.WriteLine("acdream-bake"); Console.WriteLine($"dat dir: {options.DatDir}"); Console.WriteLine($"out: {options.OutPath}"); Console.WriteLine($"threads: {options.Threads}"); Console.WriteLine(); - var sw = Stopwatch.StartNew(); + var stopwatch = Stopwatch.StartNew(); using var dats = new DatCollection(options.DatDir, DatAccessType.Read); - // The unified AcDream.Content.DatCollectionAdapter (MP1b review - // finding 7): same instance class the client and the equivalence - // suite use — adapter drift between bake and runtime is impossible - // by construction. using var datReaderWriter = new DatCollectionAdapter(dats); var extractorLogger = new ConsoleErrorLogger(nameof(MeshExtractor)); - - // Thread-safe side-stage sink for particle-preload meshes MeshExtractor - // emits mid-extraction (MP1a documented contract; extractor is shared - // across the batch workers). var sideStaged = new ConcurrentQueue(); - var extractor = new MeshExtractor(datReaderWriter, extractorLogger, data => sideStaged.Enqueue(data)); + var extractor = new MeshExtractor( + datReaderWriter, + extractorLogger, + data => sideStaged.Enqueue(data)); - // ---- enumeration ----------------------------------------------------- + var failures = new ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)>(); - var gfxObjIds = dats.GetAllIdsOfType().ToList(); - var setupIds = dats.GetAllIdsOfType().ToList(); + // ---- enumeration and content-identity catalog ---------------------- + + var gfxObjIds = dats.GetAllIdsOfType().OrderBy(id => id).ToList(); + var setupIds = dats.GetAllIdsOfType().OrderBy(id => id).ToList(); var envCellIds = EnumerateEnvCellIds(dats, options.LandblockFilter); - if (options.IdFilter is not null) { + if (options.IdFilter is not null) + { gfxObjIds = gfxObjIds.Where(options.IdFilter.Contains).ToList(); setupIds = setupIds.Where(options.IdFilter.Contains).ToList(); envCellIds = envCellIds.Where(options.IdFilter.Contains).ToList(); } - // Full work list sorted by pak key — the batching below preserves this - // global order on disk (determinism precondition). - var work = new List<(ulong Key, PakAssetType Type, uint FileId)>(gfxObjIds.Count + setupIds.Count + envCellIds.Count); - work.AddRange(gfxObjIds.Select(id => (PakKey.Compose(PakAssetType.GfxObjMesh, id), PakAssetType.GfxObjMesh, id))); - work.AddRange(setupIds.Select(id => (PakKey.Compose(PakAssetType.SetupMesh, id), PakAssetType.SetupMesh, id))); - work.AddRange(envCellIds.Select(id => (PakKey.Compose(PakAssetType.EnvCellMesh, id), PakAssetType.EnvCellMesh, id))); - work.Sort((a, b) => a.Key.CompareTo(b.Key)); + var envCellSources = new List(envCellIds.Count); + foreach (uint fileId in envCellIds) + { + cancellationToken.ThrowIfCancellationRequested(); + if (!datReaderWriter.Cell.TryGet(fileId, out var envCell) || envCell is null) + { + failures.Add((PakAssetType.EnvCellMesh, fileId, "EnvCell DAT record was not found")); + continue; + } - Console.WriteLine($"enumerated: {gfxObjIds.Count:N0} GfxObj, {setupIds.Count:N0} Setup, {envCellIds.Count:N0} EnvCell " + - $"({work.Count:N0} total)"); + envCellSources.Add( + new EnvCellBakeSource( + fileId, + envCell.EnvironmentId, + envCell.CellStructure, + envCell.Surfaces)); + } + + var envCatalog = EnvCellBakeCatalog.Build(envCellSources); + var ordinaryWork = new List<(ulong Key, PakAssetType Type, uint FileId)>( + gfxObjIds.Count + setupIds.Count); + ordinaryWork.AddRange( + gfxObjIds.Select( + id => (PakKey.Compose(PakAssetType.GfxObjMesh, id), PakAssetType.GfxObjMesh, id))); + ordinaryWork.AddRange( + setupIds.Select( + id => (PakKey.Compose(PakAssetType.SetupMesh, id), PakAssetType.SetupMesh, id))); + ordinaryWork.Sort((a, b) => a.Key.CompareTo(b.Key)); + + Console.WriteLine( + $"enumerated: {gfxObjIds.Count:N0} GfxObj, {setupIds.Count:N0} Setup, " + + $"{envCellIds.Count:N0} EnvCell"); + Console.WriteLine( + $"EnvCell catalog: {envCatalog.CellCount:N0} valid cells -> " + + $"{envCatalog.UniqueGeometryCount:N0} unique geometries + " + + $"{envCatalog.AliasCount:N0} aliases " + + $"({(envCatalog.UniqueGeometryCount == 0 ? 0 : (double)envCatalog.CellCount / envCatalog.UniqueGeometryCount):F1}x)"); Console.WriteLine(); - // ---- sorted-batch extraction + streaming write ------------------------- + // ---- sorted-batch extraction and deterministic write --------------- - var header = new PakHeader { + var header = new PakHeader + { PortalIteration = (uint)dats.Portal.Iteration!.CurrentIteration, CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration, HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration, @@ -100,153 +150,328 @@ public static class BakeRunner { BakeToolVersion = 1, }; - var failures = new ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)>(); var writtenKeys = new HashSet(); var sideStagedByKey = new Dictionary(); long completed = 0; - int writtenCounts0 = 0, writtenCounts1 = 0, writtenCounts2 = 0; // GfxObj, Setup, EnvCell + int totalExtractionJobs = ordinaryWork.Count + envCatalog.UniqueGeometryCount; + int gfxObjWritten = 0; + int setupWritten = 0; + int envCellKeysWritten = 0; + int envCellGeometriesWritten = 0; var lastProgressReport = Stopwatch.StartNew(); - using (var writer = new PakWriter(options.OutPath, header)) { - for (int batchStart = 0; batchStart < work.Count; batchStart += BatchSize) { - var batch = work.Skip(batchStart).Take(BatchSize).ToList(); - var batchResults = new ConcurrentBag<(ulong Key, PakAssetType Type, ObjectMeshData Data)>(); + using (var writer = new PakWriter(options.OutPath, header)) + { + for (int batchStart = 0; batchStart < ordinaryWork.Count; batchStart += BatchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + var batch = ordinaryWork.Skip(batchStart).Take(BatchSize).ToArray(); + var batchResults = + new ConcurrentBag<(ulong Key, PakAssetType Type, ObjectMeshData Data)>(); Parallel.ForEach( batch, - new ParallelOptions { MaxDegreeOfParallelism = options.Threads }, - item => { + new ParallelOptions + { + MaxDegreeOfParallelism = options.Threads, + CancellationToken = cancellationToken, + }, + item => + { var (key, type, fileId) = item; - try { - // EnvCell entries store the cell's synthetic geometry mesh - // (bit 32 set — see MeshExtractor.PrepareMeshData's EnvCell - // branch); the cell's static objects are covered by their - // own GfxObj/Setup entries. - ulong extractorId = type == PakAssetType.EnvCellMesh ? fileId | 0x1_0000_0000UL : fileId; - // isSetup: matches the runtime's own request sites — - // WbMeshAdapter.IncrementRefCount/EnsureLoaded pass - // isSetup: false for every MeshRef id (incl. cell-geometry - // ids); only true Setup extraction passes true. (Review - // finding 10 — keep aligned with PakEquivalenceTests.) + try + { bool isSetup = type == PakAssetType.SetupMesh; - var data = extractor.PrepareMeshData(extractorId, isSetup); - if (data is not null) { + var data = extractor.PrepareMeshData(fileId, isSetup, cancellationToken); + if (data is not null) batchResults.Add((key, type, data)); - } - else { - failures.Add((type, fileId, "extractor returned null (no polygons or unresolvable id)")); - } + else + failures.Add((type, fileId, "extractor returned null")); } - catch (Exception ex) { - // A malformed dat entry skips that id — never fatal to the - // bake, matching the runtime's own per-id behavior. - failures.Add((type, fileId, ex.Message)); + catch (OperationCanceledException) + { + throw; + } + catch (Exception exception) + { + failures.Add((type, fileId, exception.Message)); + } + finally + { + Interlocked.Increment(ref completed); } - - Interlocked.Increment(ref completed); }); - // Sequential sorted write: batch results in key order. - foreach (var (key, type, data) in batchResults.OrderBy(r => r.Key)) { + foreach (var (key, type, data) in batchResults.OrderBy(result => result.Key)) + { writer.AddBlob(key, data); writtenKeys.Add(key); - switch (type) { - case PakAssetType.GfxObjMesh: writtenCounts0++; break; - case PakAssetType.SetupMesh: writtenCounts1++; break; - case PakAssetType.EnvCellMesh: writtenCounts2++; break; + if (type == PakAssetType.GfxObjMesh) + gfxObjWritten++; + else + setupWritten++; + } + + DrainSideStaged(sideStaged, sideStagedByKey); + ReportProgressIfDue( + completed, + totalExtractionJobs, + failures.Count, + stopwatch.Elapsed, + lastProgressReport, + batchStart + BatchSize >= ordinaryWork.Count && + envCatalog.UniqueGeometryCount == 0); + } + + for (int batchStart = 0; batchStart < envCatalog.Groups.Count; batchStart += BatchSize) + { + cancellationToken.ThrowIfCancellationRequested(); + var batch = envCatalog.Groups.Skip(batchStart).Take(BatchSize).ToArray(); + var batchResults = + new ConcurrentBag<(EnvCellGeometryGroup Group, ObjectMeshData Data)>(); + + Parallel.ForEach( + batch, + new ParallelOptions + { + MaxDegreeOfParallelism = options.Threads, + CancellationToken = cancellationToken, + }, + group => + { + try + { + uint environmentDid = 0x0D00_0000u | group.EnvironmentId; + if (!datReaderWriter.Portal.TryGet( + environmentDid, + out var environment) || + environment is null || + !environment.Cells.TryGetValue( + group.CellStructure, + out var cellStruct)) + { + failures.Add(( + PakAssetType.EnvCellMesh, + group.FileIds[0], + $"environment 0x{environmentDid:X8} cell structure " + + $"{group.CellStructure} was not found")); + return; + } + + var data = extractor.PrepareCellStructMeshData( + group.GeometryId, + cellStruct, + group.Surfaces, + Matrix4x4.Identity, + cancellationToken); + if (data is not null) + batchResults.Add((group, data)); + else + failures.Add(( + PakAssetType.EnvCellMesh, + group.FileIds[0], + "unique cell-structure extraction returned null")); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception exception) + { + failures.Add(( + PakAssetType.EnvCellMesh, + group.FileIds[0], + exception.Message)); + } + finally + { + Interlocked.Increment(ref completed); + } + }); + + foreach (var (group, data) in batchResults.OrderBy(result => result.Group.FileIds[0])) + { + ulong primaryKey = + PakKey.Compose(PakAssetType.EnvCellMesh, group.FileIds[0]); + writer.AddBlob(primaryKey, data); + writtenKeys.Add(primaryKey); + envCellKeysWritten++; + envCellGeometriesWritten++; + + for (int i = 1; i < group.FileIds.Length; i++) + { + ulong aliasKey = + PakKey.Compose(PakAssetType.EnvCellMesh, group.FileIds[i]); + writer.AddAlias(aliasKey, primaryKey); + writtenKeys.Add(aliasKey); + envCellKeysWritten++; } } - // Drain side-staged preloads per batch into the dedup map so the - // queue never grows past one batch's emissions (memory bound); - // written after all batches, sorted (see below). - while (sideStaged.TryDequeue(out var staged)) { - uint fileId = (uint)(staged.ObjectId & 0xFFFFFFFFu); - ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId); - if (!sideStagedByKey.ContainsKey(key)) sideStagedByKey[key] = staged; - } - - if (lastProgressReport.Elapsed.TotalSeconds >= 5 || batchStart + BatchSize >= work.Count) { - ReportProgress(completed, work.Count, failures.Count, sw.Elapsed); - lastProgressReport.Restart(); - } + DrainSideStaged(sideStaged, sideStagedByKey); + ReportProgressIfDue( + completed, + totalExtractionJobs, + failures.Count, + stopwatch.Elapsed, + lastProgressReport, + batchStart + BatchSize >= envCatalog.Groups.Count); } - // Side-staged preload meshes not already covered by a primary entry: - // sorted by key for deterministic placement. - int sideStagedWritten = 0, sideStagedDuped = 0; - foreach (var (key, data) in sideStagedByKey.OrderBy(kv => kv.Key)) { - if (writtenKeys.Contains(key)) { sideStagedDuped++; continue; } + int sideStagedWritten = 0; + int sideStagedDuped = 0; + foreach (var (key, data) in sideStagedByKey.OrderBy(pair => pair.Key)) + { + cancellationToken.ThrowIfCancellationRequested(); + if (writtenKeys.Contains(key)) + { + sideStagedDuped++; + continue; + } + writer.AddBlob(key, data); writtenKeys.Add(key); sideStagedWritten++; } writer.Finish(); - sw.Stop(); + stopwatch.Stop(); - var outSize = new FileInfo(options.OutPath).Length; - Console.WriteLine(); - Console.WriteLine("=== bake summary ==="); - Console.WriteLine($" GfxObj baked: {writtenCounts0:N0}"); - Console.WriteLine($" Setup baked: {writtenCounts1:N0}"); - Console.WriteLine($" EnvCell baked: {writtenCounts2:N0}"); - Console.WriteLine($" side-staged baked (particle preload GfxObjs): {sideStagedWritten:N0} ({sideStagedDuped:N0} deduped)"); - Console.WriteLine($" total blobs: {writtenKeys.Count:N0}"); - Console.WriteLine($" failures: {failures.Count:N0}"); - Console.WriteLine($" elapsed: {sw.Elapsed.TotalSeconds:F1} s"); - Console.WriteLine($" output size: {outSize / 1024.0 / 1024.0:F1} MB"); - Console.WriteLine($" output path: {options.OutPath}"); + var report = new BakeReport + { + GfxObjKeys = gfxObjWritten, + SetupKeys = setupWritten, + EnvCellKeys = envCellKeysWritten, + UniqueEnvCellGeometries = envCellGeometriesWritten, + EnvCellAliases = envCellKeysWritten - envCellGeometriesWritten, + SideStagedKeys = sideStagedWritten, + PhysicalBlobs = + gfxObjWritten + setupWritten + envCellGeometriesWritten + sideStagedWritten, + TotalKeys = writtenKeys.Count, + Failures = failures.Count, + Elapsed = stopwatch.Elapsed, + OutputBytes = new FileInfo(options.OutPath).Length, + PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64, + }; + + PrintSummary(report, sideStagedDuped, options.OutPath); + PrintFailures(failures); + return report; } - - if (!failures.IsEmpty) { - Console.WriteLine(); - Console.WriteLine($"failures ({failures.Count}):"); - foreach (var (type, fileId, reason) in failures.OrderBy(f => f.FileId).Take(200)) { - Console.WriteLine($" {type,-12} 0x{fileId:X8}: {reason}"); - } - if (failures.Count > 200) Console.WriteLine($" ... and {failures.Count - 200} more"); - } - - return 0; } - private static void ReportProgress(long done, int total, int failures, TimeSpan elapsed) { + private static void DrainSideStaged( + ConcurrentQueue sideStaged, + Dictionary sideStagedByKey) + { + while (sideStaged.TryDequeue(out var staged)) + { + uint fileId = (uint)(staged.ObjectId & 0xFFFF_FFFFu); + ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId); + sideStagedByKey.TryAdd(key, staged); + } + } + + private static void ReportProgressIfDue( + long done, + int total, + int failures, + TimeSpan elapsed, + Stopwatch lastProgressReport, + bool final) + { + if (!final && lastProgressReport.Elapsed.TotalSeconds < 5) + return; + double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0; double etaSeconds = rate > 0 ? (total - done) / rate : 0; - Console.WriteLine($"[{elapsed:hh\\:mm\\:ss}] baked {done:N0}/{total:N0}, failures={failures:N0}, " + - $"elapsed={elapsed.TotalSeconds:F0}s, ETA={etaSeconds:F0}s"); + Console.WriteLine( + $"[{elapsed:hh\\:mm\\:ss}] extracted {done:N0}/{total:N0}, " + + $"failures={failures:N0}, elapsed={elapsed.TotalSeconds:F0}s, " + + $"ETA={etaSeconds:F0}s"); + lastProgressReport.Restart(); + } + + private static void PrintSummary(BakeReport report, int sideStagedDuped, string outputPath) + { + Console.WriteLine(); + Console.WriteLine("=== bake summary ==="); + Console.WriteLine($" GfxObj keys: {report.GfxObjKeys:N0}"); + Console.WriteLine($" Setup keys: {report.SetupKeys:N0}"); + Console.WriteLine($" EnvCell keys: {report.EnvCellKeys:N0}"); + Console.WriteLine($" EnvCell unique: {report.UniqueEnvCellGeometries:N0}"); + Console.WriteLine($" EnvCell aliases: {report.EnvCellAliases:N0}"); + Console.WriteLine($" EnvCell dedup: {report.EnvCellDedupRatio:F1}x"); + Console.WriteLine( + $" side-staged keys: {report.SideStagedKeys:N0} " + + $"({sideStagedDuped:N0} duplicate keys)"); + Console.WriteLine($" physical blobs: {report.PhysicalBlobs:N0}"); + Console.WriteLine($" total keys: {report.TotalKeys:N0}"); + Console.WriteLine($" failures: {report.Failures:N0}"); + Console.WriteLine($" elapsed: {report.Elapsed.TotalSeconds:F1} s"); + Console.WriteLine( + $" peak working set: {report.PeakWorkingSetBytes / 1024.0 / 1024.0:F1} MB"); + Console.WriteLine($" output size: {report.OutputBytes / 1024.0 / 1024.0:F1} MB"); + Console.WriteLine($" output path: {outputPath}"); + } + + private static void PrintFailures( + ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)> failures) + { + if (failures.IsEmpty) + return; + + Console.WriteLine(); + Console.WriteLine($"failures ({failures.Count}):"); + foreach (var (type, fileId, reason) in failures + .OrderBy(failure => failure.Type) + .ThenBy(failure => failure.FileId) + .Take(200)) + { + Console.WriteLine($" {type,-12} 0x{fileId:X8}: {reason}"); + } + + if (failures.Count > 200) + Console.WriteLine($" ... and {failures.Count - 200} more"); } /// - /// Enumerates EnvCell ids by walking the cell dat's LandBlockInfo entries - /// (0xFFFE low 16 bits) and, for each, the NumCells-derived cell id range — - /// GetAllIdsOfType<T>() does not cover cell-dat range-based types - /// (DatReaderWriter's documented limitation; see the low-16-bit bucketing - /// idiom in src/AcDream.Cli/Program.cs's CountCellByLow16, and the - /// firstCellId/NumCells hydration idiom in GameWindow.BuildPhysicsDatBundle - /// / BuildInteriorEntitiesForStreaming). + /// Enumerates EnvCell IDs by walking each cell DAT LandBlockInfo record's + /// NumCells range. DatReaderWriter cannot enumerate these range-shaped IDs + /// through GetAllIdsOfType. /// - private static List EnumerateEnvCellIds(DatCollection dats, HashSet? landblockFilter) { + private static List EnumerateEnvCellIds( + DatCollection dats, + HashSet? landblockFilter) + { var landblockInfoIds = new List(); - foreach (var file in dats.Cell.Tree) { - if ((file.Id & 0xFFFFu) != 0xFFFEu) continue; - uint landblockId = file.Id & 0xFFFF0000u; - if (landblockFilter is not null && !landblockFilter.Contains(landblockId)) continue; + foreach (var file in dats.Cell.Tree) + { + if ((file.Id & 0xFFFFu) != 0xFFFEu) + continue; + + uint landblockId = file.Id & 0xFFFF_0000u; + if (landblockFilter is not null && !landblockFilter.Contains(landblockId)) + continue; landblockInfoIds.Add(file.Id); } + landblockInfoIds.Sort(); var envCellIds = new List(); - foreach (var lbInfoId in landblockInfoIds) { - if (!dats.Cell.TryGet(lbInfoId, out var lbInfo) || lbInfo is null) continue; - if (lbInfo.NumCells == 0) continue; - - uint landblockId = lbInfoId & 0xFFFF0000u; - uint firstCellId = landblockId | 0x0100u; - for (uint offset = 0; offset < lbInfo.NumCells; offset++) { - envCellIds.Add(firstCellId + offset); + foreach (uint landblockInfoId in landblockInfoIds) + { + if (!dats.Cell.TryGet(landblockInfoId, out var info) || + info is null || + info.NumCells == 0) + { + continue; } + + uint firstCellId = (landblockInfoId & 0xFFFF_0000u) | 0x0100u; + for (uint offset = 0; offset < info.NumCells; offset++) + envCellIds.Add(firstCellId + offset); } + return envCellIds; } } diff --git a/src/AcDream.Bake/EnvCellBakeCatalog.cs b/src/AcDream.Bake/EnvCellBakeCatalog.cs new file mode 100644 index 00000000..5665718b --- /dev/null +++ b/src/AcDream.Bake/EnvCellBakeCatalog.cs @@ -0,0 +1,119 @@ +using AcDream.Core.Rendering.Wb; + +namespace AcDream.Bake; + +/// +/// The identity-bearing portion of one EnvCell DAT record needed by the bake. +/// Position, portals, and static objects are instance data and intentionally do +/// not participate in shared shell geometry. +/// +public sealed record EnvCellBakeSource( + uint FileId, + uint EnvironmentId, + ushort CellStructure, + IReadOnlyList Surfaces); + +/// One unique shell geometry and every independently addressable cell key that uses it. +public sealed record EnvCellGeometryGroup( + ulong GeometryId, + uint EnvironmentId, + ushort CellStructure, + ushort[] Surfaces, + uint[] FileIds); + +/// +/// Builds the deterministic file-id alias map used by the offline bake. +/// Geometry equality is checked against the complete source tuple so a hash +/// collision can never silently merge different cells. +/// +public sealed class EnvCellBakeCatalog +{ + public IReadOnlyList Groups { get; } + public int CellCount { get; } + public int UniqueGeometryCount => Groups.Count; + public int AliasCount => CellCount - UniqueGeometryCount; + + private EnvCellBakeCatalog(IReadOnlyList groups, int cellCount) + { + Groups = groups; + CellCount = cellCount; + } + + public static EnvCellBakeCatalog Build(IEnumerable sources) => + Build(sources, EnvCellGeometryIdentity.Compute); + + /// + /// Identity injection exists for collision-path conformance tests. Production + /// always calls the overload bound to . + /// + public static EnvCellBakeCatalog Build( + IEnumerable sources, + Func, ulong> computeIdentity) + { + ArgumentNullException.ThrowIfNull(sources); + ArgumentNullException.ThrowIfNull(computeIdentity); + + var byGeometry = new Dictionary(); + var fileIds = new HashSet(); + int cellCount = 0; + + foreach (var source in sources.OrderBy(source => source.FileId)) + { + if (!fileIds.Add(source.FileId)) + throw new InvalidDataException($"duplicate EnvCell file id 0x{source.FileId:X8}"); + + var surfaces = source.Surfaces.ToArray(); + ulong geometryId = computeIdentity( + source.EnvironmentId, + source.CellStructure, + surfaces); + + if (byGeometry.TryGetValue(geometryId, out var existing)) + { + if (existing.EnvironmentId != source.EnvironmentId || + existing.CellStructure != source.CellStructure || + !existing.Surfaces.AsSpan().SequenceEqual(surfaces)) + { + throw new InvalidDataException( + $"EnvCell geometry identity collision 0x{geometryId:X16}: " + + $"cell 0x{existing.FileIds[0]:X8} and cell 0x{source.FileId:X8} " + + "have different environment/cell-structure/surface tuples"); + } + + existing.FileIds.Add(source.FileId); + } + else + { + byGeometry.Add( + geometryId, + new MutableGroup( + geometryId, + source.EnvironmentId, + source.CellStructure, + surfaces, + new List { source.FileId })); + } + + cellCount++; + } + + var groups = byGeometry.Values + .Select(group => new EnvCellGeometryGroup( + group.GeometryId, + group.EnvironmentId, + group.CellStructure, + group.Surfaces, + group.FileIds.ToArray())) + .OrderBy(group => group.FileIds[0]) + .ToArray(); + + return new EnvCellBakeCatalog(groups, cellCount); + } + + private sealed record MutableGroup( + ulong GeometryId, + uint EnvironmentId, + ushort CellStructure, + ushort[] Surfaces, + List FileIds); +} diff --git a/src/AcDream.Content/MeshExtractor.cs b/src/AcDream.Content/MeshExtractor.cs index 7973304a..95b8dbda 100644 --- a/src/AcDream.Content/MeshExtractor.cs +++ b/src/AcDream.Content/MeshExtractor.cs @@ -686,7 +686,7 @@ public sealed class MeshExtractor { }; } - public ObjectMeshData? PrepareCellStructMeshData(ulong id, CellStruct cellStruct, List surfaceOverrides, Matrix4x4 transform, CancellationToken ct) { + public ObjectMeshData? PrepareCellStructMeshData(ulong id, CellStruct cellStruct, IReadOnlyList surfaceOverrides, Matrix4x4 transform, CancellationToken ct) { var vertices = new List(); var UVLookup = new Dictionary<(ushort vertId, ushort uvIdx, bool isNeg), ushort>(); var batchesByFormat = new Dictionary<(int Width, int Height, TextureFormat Format), List>(); diff --git a/tests/AcDream.Bake.Tests/EnvCellBakeCatalogTests.cs b/tests/AcDream.Bake.Tests/EnvCellBakeCatalogTests.cs new file mode 100644 index 00000000..bd0a2b42 --- /dev/null +++ b/tests/AcDream.Bake.Tests/EnvCellBakeCatalogTests.cs @@ -0,0 +1,85 @@ +namespace AcDream.Bake.Tests; + +public sealed class EnvCellBakeCatalogTests +{ + [Fact] + public void Build_GroupsEqualGeometryAndPreservesEverySortedFileId() + { + var catalog = EnvCellBakeCatalog.Build( + [ + new(0x2222_0102, 7, 3, new ushort[] { 10, 20 }), + new(0x1111_0100, 7, 3, new ushort[] { 10, 20 }), + new(0x1111_0101, 7, 4, new ushort[] { 10, 20 }), + new(0x2222_0100, 7, 3, new ushort[] { 10, 20 }), + ]); + + Assert.Equal(4, catalog.CellCount); + Assert.Equal(2, catalog.UniqueGeometryCount); + Assert.Equal(2, catalog.AliasCount); + Assert.Equal( + new uint[] { 0x1111_0100, 0x2222_0100, 0x2222_0102 }, + catalog.Groups[0].FileIds); + Assert.Equal(new uint[] { 0x1111_0101 }, catalog.Groups[1].FileIds); + } + + [Fact] + public void Build_IsDeterministicAcrossSourceOrder() + { + EnvCellBakeSource[] sources = + [ + new(4, 9, 2, new ushort[] { 5, 6 }), + new(2, 9, 2, new ushort[] { 5, 6 }), + new(3, 1, 8, new ushort[] { 7 }), + ]; + + var forward = EnvCellBakeCatalog.Build(sources); + var reverse = EnvCellBakeCatalog.Build(sources.Reverse()); + + Assert.Equal(forward.Groups, reverse.Groups, GroupComparer.Instance); + } + + [Fact] + public void Build_RejectsIdentityCollisionWithDifferentTuple() + { + EnvCellBakeSource[] sources = + [ + new(1, 10, 20, new ushort[] { 30 }), + new(2, 11, 20, new ushort[] { 30 }), + ]; + + var exception = Assert.Throws( + () => EnvCellBakeCatalog.Build(sources, (_, _, _) => 0x2_0000_0000)); + + Assert.Contains("collision", exception.Message); + Assert.Contains("0x00000001", exception.Message); + Assert.Contains("0x00000002", exception.Message); + } + + [Fact] + public void Build_RejectsDuplicateFileId() + { + EnvCellBakeSource[] sources = + [ + new(1, 10, 20, Array.Empty()), + new(1, 10, 20, Array.Empty()), + ]; + + Assert.Throws(() => EnvCellBakeCatalog.Build(sources)); + } + + private sealed class GroupComparer : IEqualityComparer + { + public static GroupComparer Instance { get; } = new(); + + public bool Equals(EnvCellGeometryGroup? x, EnvCellGeometryGroup? y) => + x is not null && + y is not null && + x.GeometryId == y.GeometryId && + x.EnvironmentId == y.EnvironmentId && + x.CellStructure == y.CellStructure && + x.Surfaces.AsSpan().SequenceEqual(y.Surfaces) && + x.FileIds.AsSpan().SequenceEqual(y.FileIds); + + public int GetHashCode(EnvCellGeometryGroup obj) => obj.GeometryId.GetHashCode(); + } +}