perf(bake): stream catalog and bound validation residency

src/AcDream.Bake/BakeArtifactValidator.cs

@@ -34,6 +34,6 @@ public static class BakeArtifactValidator
                 $"bake TOC count {actual.TocCount} does not match expected " +
                 $"{expectedTocCount}");
 
-        reader.ValidateComplete();
+        reader.ValidateTocStructure();
     }
 }

src/AcDream.Bake/BakeOutputTransaction.cs

@@ -37,10 +37,12 @@ public static class BakeOutputTransaction
             validateTemporary(temporaryPath, result);
             cancellationToken.ThrowIfCancellationRequested();
 
-            if (File.Exists(fullDestination))
+            // Same-volume MoveFileEx/rename is the publication primitive.
-                File.Replace(temporaryPath, fullDestination, null, ignoreMetadataErrors: true);
+            // File.Replace additionally performs destination metadata/backup
-            else
+            // semantics and proved brittle for a validated 28 GiB artifact on
-                File.Move(temporaryPath, fullDestination);
+            // Windows. Move(overwrite:true) changes only the directory entry:
+            // readers observe the complete old file or the complete new file.
+            File.Move(temporaryPath, fullDestination, overwrite: true);
 
             return result;
         }

src/AcDream.Bake/BakeRunner.cs

@@ -35,9 +35,11 @@ public sealed record BakeReport
     public required int PhysicalBlobs { get; init; }
     public required int TotalKeys { get; init; }
     public required int Failures { get; init; }
+    public required TimeSpan ExtractionAndWriteElapsed { get; init; }
     public required TimeSpan Elapsed { get; init; }
     public required long OutputBytes { get; init; }
     public required long PeakWorkingSetBytes { get; init; }
+    public required long PeakPrivateBytes { get; init; }
 
     public double EnvCellDedupRatio =>
         UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries;
@@ -54,7 +56,7 @@ public static class BakeRunner
     /// Items extracted in parallel per batch before sequential sorted write.
     /// Bounds peak decoded output while keeping workers busy.
     /// </summary>
-    private const int BatchSize = 512;
+    private const int BatchSize = 16;
 
     public static int Run(BakeOptions options)
     {
@@ -69,6 +71,7 @@ public static class BakeRunner
             throw new ArgumentOutOfRangeException(nameof(options), "thread count must be positive");
 
         options.CancellationToken.ThrowIfCancellationRequested();
+        var totalStopwatch = Stopwatch.StartNew();
         var report = BakeOutputTransaction.WriteValidateAndPublish(
             options.OutPath,
             temporaryPath => RunCore(
@@ -79,6 +82,15 @@ public static class BakeRunner
                 result.Header,
                 result.TotalKeys),
             options.CancellationToken);
+        totalStopwatch.Stop();
+
+        report = report with
+        {
+            Elapsed = totalStopwatch.Elapsed,
+            OutputBytes = new FileInfo(options.OutPath).Length,
+            PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
+            PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64,
+        };
 
         PrintSummary(report, options.OutPath);
         return report;
@@ -119,7 +131,7 @@ public static class BakeRunner
             envCellIds = envCellIds.Where(options.IdFilter.Contains).ToList();
         }
 
-        var envCellSources = new List<EnvCellBakeSource>(envCellIds.Count);
+        var envCatalogBuilder = new EnvCellBakeCatalogBuilder();
         foreach (uint fileId in envCellIds)
         {
             cancellationToken.ThrowIfCancellationRequested();
@@ -129,7 +141,7 @@ public static class BakeRunner
                 continue;
             }
 
-            envCellSources.Add(
+            envCatalogBuilder.Add(
                 new EnvCellBakeSource(
                     fileId,
                     envCell.EnvironmentId,
@@ -137,7 +149,7 @@ public static class BakeRunner
                     envCell.Surfaces));
         }
 
-        var envCatalog = EnvCellBakeCatalog.Build(envCellSources);
+        var envCatalog = envCatalogBuilder.Build();
         var ordinaryWork = new List<(ulong Key, PakAssetType Type, uint FileId)>(
             gfxObjIds.Count + setupIds.Count);
         ordinaryWork.AddRange(
@@ -177,6 +189,7 @@ public static class BakeRunner
         int setupWritten = 0;
         int envCellKeysWritten = 0;
         int envCellGeometriesWritten = 0;
+        int sideStagedDuped = 0;
         var lastProgressReport = Stopwatch.StartNew();
 
         using (var writer = new PakWriter(options.OutPath, header))
@@ -231,7 +244,10 @@ public static class BakeRunner
                         setupWritten++;
                 }
 
-                DrainSideStaged(sideStaged, sideStagedByKey);
+                sideStagedDuped += DrainSideStaged(
+                    sideStaged,
+                    sideStagedByKey,
+                    writtenKeys);
                 ReportProgressIfDue(
                     completed,
                     totalExtractionJobs,
@@ -327,7 +343,10 @@ public static class BakeRunner
                     }
                 }
 
-                DrainSideStaged(sideStaged, sideStagedByKey);
+                sideStagedDuped += DrainSideStaged(
+                    sideStaged,
+                    sideStagedByKey,
+                    writtenKeys);
                 ReportProgressIfDue(
                     completed,
                     totalExtractionJobs,
@@ -338,7 +357,6 @@ public static class BakeRunner
             }
 
             int sideStagedWritten = 0;
-            int sideStagedDuped = 0;
             foreach (var (key, data) in sideStagedByKey.OrderBy(pair => pair.Key))
             {
                 cancellationToken.ThrowIfCancellationRequested();
@@ -369,9 +387,11 @@ public static class BakeRunner
                     gfxObjWritten + setupWritten + envCellGeometriesWritten + sideStagedWritten,
                 TotalKeys = writtenKeys.Count,
                 Failures = failures.Count,
+                ExtractionAndWriteElapsed = stopwatch.Elapsed,
                 Elapsed = stopwatch.Elapsed,
                 OutputBytes = new FileInfo(options.OutPath).Length,
                 PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
+                PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64,
             };
 
             report = report with { SideStagedDuplicateKeys = sideStagedDuped };
@@ -380,16 +400,20 @@ public static class BakeRunner
         }
     }
 
-    private static void DrainSideStaged(
+    private static int DrainSideStaged(
         ConcurrentQueue<ObjectMeshData> sideStaged,
-        Dictionary<ulong, ObjectMeshData> sideStagedByKey)
+        Dictionary<ulong, ObjectMeshData> sideStagedByKey,
+        HashSet<ulong> writtenKeys)
     {
+        int duplicates = 0;
         while (sideStaged.TryDequeue(out var staged))
         {
             uint fileId = (uint)(staged.ObjectId & 0xFFFF_FFFFu);
             ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId);
-            sideStagedByKey.TryAdd(key, staged);
+            if (writtenKeys.Contains(key) || !sideStagedByKey.TryAdd(key, staged))
+                duplicates++;
         }
+        return duplicates;
     }
 
     private static void ReportProgressIfDue(
@@ -405,10 +429,14 @@ public static class BakeRunner
 
         double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0;
         double etaSeconds = rate > 0 ? (total - done) / rate : 0;
+        using var process = Process.GetCurrentProcess();
+        process.Refresh();
+        long managedHeap = GC.GetGCMemoryInfo().HeapSizeBytes;
         Console.WriteLine(
             $"[{elapsed:hh\\:mm\\:ss}] extracted {done:N0}/{total:N0}, " +
             $"failures={failures:N0}, elapsed={elapsed.TotalSeconds:F0}s, " +
-            $"ETA={etaSeconds:F0}s");
+            $"ETA={etaSeconds:F0}s, private={process.PrivateMemorySize64 / 1024.0 / 1024.0:F0}MB, " +
+            $"managed={managedHeap / 1024.0 / 1024.0:F0}MB");
         lastProgressReport.Restart();
     }
 
@@ -428,9 +456,13 @@ public static class BakeRunner
         Console.WriteLine($"  physical blobs:   {report.PhysicalBlobs:N0}");
         Console.WriteLine($"  total keys:       {report.TotalKeys:N0}");
         Console.WriteLine($"  failures:         {report.Failures:N0}");
-        Console.WriteLine($"  elapsed:          {report.Elapsed.TotalSeconds:F1} s");
+        Console.WriteLine(
+            $"  extract + write:  {report.ExtractionAndWriteElapsed.TotalSeconds:F1} s");
+        Console.WriteLine($"  total validated:  {report.Elapsed.TotalSeconds:F1} s");
         Console.WriteLine(
             $"  peak working set: {report.PeakWorkingSetBytes / 1024.0 / 1024.0:F1} MB");
+        Console.WriteLine(
+            $"  peak private:     {report.PeakPrivateBytes / 1024.0 / 1024.0:F1} MB");
         Console.WriteLine($"  output size:      {report.OutputBytes / 1024.0 / 1024.0:F1} MB");
         Console.WriteLine($"  output path:      {outputPath}");
     }

src/AcDream.Bake/EnvCellBakeCatalog.cs

@@ -53,51 +53,94 @@ public sealed class EnvCellBakeCatalog
         ArgumentNullException.ThrowIfNull(sources);
         ArgumentNullException.ThrowIfNull(computeIdentity);
 
-        var byGeometry = new Dictionary<ulong, MutableGroup>();
+        var builder = new EnvCellBakeCatalogBuilder(computeIdentity);
-        var fileIds = new HashSet<uint>();
+        foreach (var source in sources)
-        int cellCount = 0;
+            builder.Add(source);
+        return builder.Build();
+    }
 
-        foreach (var source in sources.OrderBy(source => source.FileId))
+    internal static EnvCellBakeCatalog Create(
+        IReadOnlyList<EnvCellGeometryGroup> groups,
+        int cellCount) =>
+        new(groups, cellCount);
+}
+
+/// <summary>
+/// Incremental catalog builder used by the full bake so 729,888 EnvCell
+/// records and their surface lists are never retained at once.
+/// </summary>
+public sealed class EnvCellBakeCatalogBuilder
+{
+    private readonly Func<uint, ushort, IReadOnlyList<ushort>, ulong> _computeIdentity;
+    private readonly Dictionary<ulong, MutableGroup> _byGeometry = new();
+    private readonly HashSet<uint> _fileIds = new();
+    private int _cellCount;
+    private bool _built;
+
+    public EnvCellBakeCatalogBuilder()
+        : this(EnvCellGeometryIdentity.Compute)
+    {
+    }
+
+    public EnvCellBakeCatalogBuilder(
+        Func<uint, ushort, IReadOnlyList<ushort>, ulong> computeIdentity)
+    {
+        ArgumentNullException.ThrowIfNull(computeIdentity);
+        _computeIdentity = computeIdentity;
+    }
+
+    public void Add(EnvCellBakeSource source)
+    {
+        if (_built)
+            throw new InvalidOperationException("the EnvCell bake catalog is already complete");
+
+        if (!_fileIds.Add(source.FileId))
+            throw new InvalidDataException($"duplicate EnvCell file id 0x{source.FileId:X8}");
+
+        ulong geometryId = _computeIdentity(
+            source.EnvironmentId,
+            source.CellStructure,
+            source.Surfaces);
+
+        if (_byGeometry.TryGetValue(geometryId, out var existing))
         {
-            if (!fileIds.Add(source.FileId))
+            if (existing.EnvironmentId != source.EnvironmentId ||
-                throw new InvalidDataException($"duplicate EnvCell file id 0x{source.FileId:X8}");
+                existing.CellStructure != source.CellStructure ||
-
+                !SurfacesEqual(existing.Surfaces, source.Surfaces))
-            var surfaces = source.Surfaces.ToArray();
-            ulong geometryId = computeIdentity(
-                source.EnvironmentId,
-                source.CellStructure,
-                surfaces);
-
-            if (byGeometry.TryGetValue(geometryId, out var existing))
             {
-                if (existing.EnvironmentId != source.EnvironmentId ||
+                throw new InvalidDataException(
-                    existing.CellStructure != source.CellStructure ||
+                    $"EnvCell geometry identity collision 0x{geometryId:X16}: " +
-                    !existing.Surfaces.AsSpan().SequenceEqual(surfaces))
+                    $"cell 0x{existing.FileIds[0]:X8} and cell 0x{source.FileId:X8} " +
-                {
+                    "have different environment/cell-structure/surface tuples");
-                    throw new InvalidDataException(
-                        $"EnvCell geometry identity collision 0x{geometryId:X16}: " +
-                        $"cell 0x{existing.FileIds[0]:X8} and cell 0x{source.FileId:X8} " +
-                        "have different environment/cell-structure/surface tuples");
-                }
-
-                existing.FileIds.Add(source.FileId);
             }
-            else
+
-            {
+            existing.FileIds.Add(source.FileId);
-                byGeometry.Add(
+        }
+        else
+        {
+            _byGeometry.Add(
+                geometryId,
+                new MutableGroup(
                     geometryId,
-                    new MutableGroup(
+                    source.EnvironmentId,
-                        geometryId,
+                    source.CellStructure,
-                        source.EnvironmentId,
+                    source.Surfaces.ToArray(),
-                        source.CellStructure,
+                    new List<uint> { source.FileId }));
-                        surfaces,
-                        new List<uint> { source.FileId }));
-            }
-
-            cellCount++;
         }
 
-        var groups = byGeometry.Values
+        _cellCount++;
+    }
+
+    public EnvCellBakeCatalog Build()
+    {
+        if (_built)
+            throw new InvalidOperationException("the EnvCell bake catalog is already complete");
+        _built = true;
+
+        foreach (var group in _byGeometry.Values)
+            group.FileIds.Sort();
+
+        var groups = _byGeometry.Values
             .Select(group => new EnvCellGeometryGroup(
                 group.GeometryId,
                 group.EnvironmentId,
@@ -107,7 +150,21 @@ public sealed class EnvCellBakeCatalog
             .OrderBy(group => group.FileIds[0])
             .ToArray();
 
-        return new EnvCellBakeCatalog(groups, cellCount);
+        return EnvCellBakeCatalog.Create(groups, _cellCount);
+    }
+
+    private static bool SurfacesEqual(
+        ReadOnlySpan<ushort> expected,
+        IReadOnlyList<ushort> actual)
+    {
+        if (expected.Length != actual.Count)
+            return false;
+        for (int i = 0; i < expected.Length; i++)
+        {
+            if (expected[i] != actual[i])
+                return false;
+        }
+        return true;
     }
 
     private sealed record MutableGroup(

src/AcDream.Content/Pak/PakReader.cs

@@ -174,13 +174,13 @@ public sealed class PakReader : IDisposable {
     }
 
     /// <summary>
-    /// Validates the complete immutable artifact before an offline bake
+    /// Validates the complete immutable TOC before an offline bake publishes
-    /// publishes it. This is intentionally stricter than runtime lookup:
+    /// it. This is intentionally stricter than runtime lookup: key order and
-    /// TOC order/ranges and the CRC of every unique physical blob must all be
+    /// every physical range must be valid. Blob CRC remains the reader's lazy
-    /// valid. Aliases are checked once per shared receipt.
+    /// lookup tripwire; scanning a multi-gigabyte mmap here would fault the
+    /// whole pak into the process working set just before publication.
     /// </summary>
-    public void ValidateComplete() {
+    public void ValidateTocStructure() {
-        var validatedReceipts = new HashSet<(ulong Offset, uint Length, uint Crc32)>();
         ulong previousKey = 0;
 
         for (int i = 0; i < _toc.Length; i++) {
@@ -198,19 +198,6 @@ public sealed class PakReader : IDisposable {
                     $"(offset={entry.Offset}, length={entry.Length}, " +
                     $"file={_fileLength}, toc@{Header.TocOffset})");
             }
-
-            var receipt = (entry.Offset, entry.Length, entry.Crc32);
-            if (!validatedReceipts.Add(receipt))
-                continue;
-
-            var bytes = new byte[entry.Length];
-            _accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length);
-            uint actualCrc = Crc32.Compute(bytes);
-            if (actualCrc != entry.Crc32) {
-                throw new InvalidDataException(
-                    $"pak blob 0x{entry.Key:X16} failed CRC validation: " +
-                    $"expected 0x{entry.Crc32:X8}, got 0x{actualCrc:X8}");
-            }
         }
     }
 

tests/AcDream.Bake.Tests/BakeDeterminismTests.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.IO;
 using AcDream.Bake;
+using AcDream.Content.Pak;
 
 namespace AcDream.Bake.Tests;
 
@@ -76,4 +77,59 @@ public sealed class BakeDeterminismTests : IDisposable {
         Assert.True(bytesA.AsSpan().SequenceEqual(bytesB),
             "two bakes of the same id set produced different bytes — the sorted-batching determinism guarantee is broken");
     }
+
+    [Fact]
+    public void Bake_RealDuplicateCells_ShareOneBlobAcrossLandblocksAndThreads() {
+        var datDir = ResolveDatDir();
+        if (datDir is null) return;
+
+        // Full-bake discovery fixture: these eight cells span multiple
+        // landblocks but have the same complete environment/structure/surface
+        // tuple. They must remain eight addressable keys backed by one blob.
+        var ids = new HashSet<uint> {
+            0x00A90230u, 0x00B901A8u, 0x00E50602u, 0x00E50610u,
+            0x00E5061Fu, 0x00F90784u, 0x00F9078Eu, 0x00FA073Eu,
+        };
+        var pathA = NewTempPakPath();
+        var pathB = NewTempPakPath();
+
+        var reportA = BakeRunner.RunDetailed(
+            new BakeOptions {
+                DatDir = datDir,
+                OutPath = pathA,
+                IdFilter = ids,
+                Threads = 1,
+            });
+        var reportB = BakeRunner.RunDetailed(
+            new BakeOptions {
+                DatDir = datDir,
+                OutPath = pathB,
+                IdFilter = ids,
+                Threads = 8,
+            });
+
+        Assert.Equal(8, reportA.EnvCellKeys);
+        Assert.Equal(1, reportA.UniqueEnvCellGeometries);
+        Assert.Equal(7, reportA.EnvCellAliases);
+        Assert.Equal(1, reportA.PhysicalBlobs);
+        Assert.Equal(reportA.TotalKeys, reportB.TotalKeys);
+        Assert.True(File.ReadAllBytes(pathA).AsSpan().SequenceEqual(File.ReadAllBytes(pathB)));
+
+        using var reader = new PakReader(pathA);
+        var receipts = ids
+            .Select(id => reader.GetTocEntryForTest(
+                PakKey.Compose(PakAssetType.EnvCellMesh, id)))
+            .ToArray();
+        Assert.All(receipts, receipt => Assert.Equal(receipts[0].Offset, receipt.Offset));
+        Assert.All(receipts, receipt => Assert.Equal(receipts[0].Length, receipt.Length));
+        Assert.All(receipts, receipt => Assert.Equal(receipts[0].Crc32, receipt.Crc32));
+
+        Assert.True(
+            reader.TryReadObjectMeshData(
+                PakKey.Compose(PakAssetType.EnvCellMesh, ids.Min()),
+                out var geometry));
+        Assert.NotNull(geometry);
+        Assert.Equal(0xE000_0000_0000_0000UL,
+            geometry.ObjectId & 0xF000_0000_0000_0000UL);
+    }
 }

tests/AcDream.Bake.Tests/BakeOutputTransactionTests.cs

@@ -122,7 +122,7 @@ public sealed class BakeOutputTransactionTests : IDisposable
     }
 
     [Fact]
-    public void ArtifactValidator_RejectsWrongDatIterationAndCorruptBlob()
+    public void ArtifactValidator_RejectsWrongDatIterationAndCorruptTocRange()
     {
         string pak = Path.Combine(_directory, "content.pak");
         var expected = Header();
@@ -139,16 +139,15 @@ public sealed class BakeOutputTransactionTests : IDisposable
         Assert.Throws<InvalidDataException>(
             () => BakeArtifactValidator.Validate(pak, wrongIteration, expectedTocCount: 1));
 
-        long offset;
-        using (var reader = new PakReader(pak))
-            offset = reader.GetBlobOffsetForTest(
-                PakKey.Compose(PakAssetType.GfxObjMesh, 1));
         using (var stream = new FileStream(pak, FileMode.Open, FileAccess.ReadWrite))
         {
-            stream.Position = offset;
+            var header = PakHeader.ReadFrom(stream);
-            int value = stream.ReadByte();
+            stream.Position = checked((long)header.TocOffset + 16);
-            stream.Position = offset;
+            Span<byte> invalidLength = stackalloc byte[4];
-            stream.WriteByte((byte)(value ^ 0xFF));
+            System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(
+                invalidLength,
+                uint.MaxValue);
+            stream.Write(invalidLength);
         }
 
         Assert.Throws<InvalidDataException>(