diff --git a/src/AcDream.Bake/BakeArtifactValidator.cs b/src/AcDream.Bake/BakeArtifactValidator.cs index d3a5950d..ceb23edd 100644 --- a/src/AcDream.Bake/BakeArtifactValidator.cs +++ b/src/AcDream.Bake/BakeArtifactValidator.cs @@ -34,6 +34,6 @@ public static class BakeArtifactValidator $"bake TOC count {actual.TocCount} does not match expected " + $"{expectedTocCount}"); - reader.ValidateComplete(); + reader.ValidateTocStructure(); } } diff --git a/src/AcDream.Bake/BakeOutputTransaction.cs b/src/AcDream.Bake/BakeOutputTransaction.cs index 4dd42640..0bc58e16 100644 --- a/src/AcDream.Bake/BakeOutputTransaction.cs +++ b/src/AcDream.Bake/BakeOutputTransaction.cs @@ -37,10 +37,12 @@ public static class BakeOutputTransaction validateTemporary(temporaryPath, result); cancellationToken.ThrowIfCancellationRequested(); - if (File.Exists(fullDestination)) - File.Replace(temporaryPath, fullDestination, null, ignoreMetadataErrors: true); - else - File.Move(temporaryPath, fullDestination); + // Same-volume MoveFileEx/rename is the publication primitive. + // File.Replace additionally performs destination metadata/backup + // semantics and proved brittle for a validated 28 GiB artifact on + // Windows. Move(overwrite:true) changes only the directory entry: + // readers observe the complete old file or the complete new file. + File.Move(temporaryPath, fullDestination, overwrite: true); return result; } diff --git a/src/AcDream.Bake/BakeRunner.cs b/src/AcDream.Bake/BakeRunner.cs index 4b4e4939..f2346bc4 100644 --- a/src/AcDream.Bake/BakeRunner.cs +++ b/src/AcDream.Bake/BakeRunner.cs @@ -35,9 +35,11 @@ public sealed record BakeReport public required int PhysicalBlobs { get; init; } public required int TotalKeys { get; init; } public required int Failures { get; init; } + public required TimeSpan ExtractionAndWriteElapsed { get; init; } public required TimeSpan Elapsed { get; init; } public required long OutputBytes { get; init; } public required long PeakWorkingSetBytes { get; init; } + public required long PeakPrivateBytes { get; init; } public double EnvCellDedupRatio => UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries; @@ -54,7 +56,7 @@ public static class BakeRunner /// Items extracted in parallel per batch before sequential sorted write. /// Bounds peak decoded output while keeping workers busy. /// - private const int BatchSize = 512; + private const int BatchSize = 16; public static int Run(BakeOptions options) { @@ -69,6 +71,7 @@ public static class BakeRunner throw new ArgumentOutOfRangeException(nameof(options), "thread count must be positive"); options.CancellationToken.ThrowIfCancellationRequested(); + var totalStopwatch = Stopwatch.StartNew(); var report = BakeOutputTransaction.WriteValidateAndPublish( options.OutPath, temporaryPath => RunCore( @@ -79,6 +82,15 @@ public static class BakeRunner result.Header, result.TotalKeys), options.CancellationToken); + totalStopwatch.Stop(); + + report = report with + { + Elapsed = totalStopwatch.Elapsed, + OutputBytes = new FileInfo(options.OutPath).Length, + PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64, + PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64, + }; PrintSummary(report, options.OutPath); return report; @@ -119,7 +131,7 @@ public static class BakeRunner envCellIds = envCellIds.Where(options.IdFilter.Contains).ToList(); } - var envCellSources = new List(envCellIds.Count); + var envCatalogBuilder = new EnvCellBakeCatalogBuilder(); foreach (uint fileId in envCellIds) { cancellationToken.ThrowIfCancellationRequested(); @@ -129,7 +141,7 @@ public static class BakeRunner continue; } - envCellSources.Add( + envCatalogBuilder.Add( new EnvCellBakeSource( fileId, envCell.EnvironmentId, @@ -137,7 +149,7 @@ public static class BakeRunner envCell.Surfaces)); } - var envCatalog = EnvCellBakeCatalog.Build(envCellSources); + var envCatalog = envCatalogBuilder.Build(); var ordinaryWork = new List<(ulong Key, PakAssetType Type, uint FileId)>( gfxObjIds.Count + setupIds.Count); ordinaryWork.AddRange( @@ -177,6 +189,7 @@ public static class BakeRunner int setupWritten = 0; int envCellKeysWritten = 0; int envCellGeometriesWritten = 0; + int sideStagedDuped = 0; var lastProgressReport = Stopwatch.StartNew(); using (var writer = new PakWriter(options.OutPath, header)) @@ -231,7 +244,10 @@ public static class BakeRunner setupWritten++; } - DrainSideStaged(sideStaged, sideStagedByKey); + sideStagedDuped += DrainSideStaged( + sideStaged, + sideStagedByKey, + writtenKeys); ReportProgressIfDue( completed, totalExtractionJobs, @@ -327,7 +343,10 @@ public static class BakeRunner } } - DrainSideStaged(sideStaged, sideStagedByKey); + sideStagedDuped += DrainSideStaged( + sideStaged, + sideStagedByKey, + writtenKeys); ReportProgressIfDue( completed, totalExtractionJobs, @@ -338,7 +357,6 @@ public static class BakeRunner } int sideStagedWritten = 0; - int sideStagedDuped = 0; foreach (var (key, data) in sideStagedByKey.OrderBy(pair => pair.Key)) { cancellationToken.ThrowIfCancellationRequested(); @@ -369,9 +387,11 @@ public static class BakeRunner gfxObjWritten + setupWritten + envCellGeometriesWritten + sideStagedWritten, TotalKeys = writtenKeys.Count, Failures = failures.Count, + ExtractionAndWriteElapsed = stopwatch.Elapsed, Elapsed = stopwatch.Elapsed, OutputBytes = new FileInfo(options.OutPath).Length, PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64, + PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64, }; report = report with { SideStagedDuplicateKeys = sideStagedDuped }; @@ -380,16 +400,20 @@ public static class BakeRunner } } - private static void DrainSideStaged( + private static int DrainSideStaged( ConcurrentQueue sideStaged, - Dictionary sideStagedByKey) + Dictionary sideStagedByKey, + HashSet writtenKeys) { + int duplicates = 0; while (sideStaged.TryDequeue(out var staged)) { uint fileId = (uint)(staged.ObjectId & 0xFFFF_FFFFu); ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId); - sideStagedByKey.TryAdd(key, staged); + if (writtenKeys.Contains(key) || !sideStagedByKey.TryAdd(key, staged)) + duplicates++; } + return duplicates; } private static void ReportProgressIfDue( @@ -405,10 +429,14 @@ public static class BakeRunner double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0; double etaSeconds = rate > 0 ? (total - done) / rate : 0; + using var process = Process.GetCurrentProcess(); + process.Refresh(); + long managedHeap = GC.GetGCMemoryInfo().HeapSizeBytes; Console.WriteLine( $"[{elapsed:hh\\:mm\\:ss}] extracted {done:N0}/{total:N0}, " + $"failures={failures:N0}, elapsed={elapsed.TotalSeconds:F0}s, " + - $"ETA={etaSeconds:F0}s"); + $"ETA={etaSeconds:F0}s, private={process.PrivateMemorySize64 / 1024.0 / 1024.0:F0}MB, " + + $"managed={managedHeap / 1024.0 / 1024.0:F0}MB"); lastProgressReport.Restart(); } @@ -428,9 +456,13 @@ public static class BakeRunner Console.WriteLine($" physical blobs: {report.PhysicalBlobs:N0}"); Console.WriteLine($" total keys: {report.TotalKeys:N0}"); Console.WriteLine($" failures: {report.Failures:N0}"); - Console.WriteLine($" elapsed: {report.Elapsed.TotalSeconds:F1} s"); + Console.WriteLine( + $" extract + write: {report.ExtractionAndWriteElapsed.TotalSeconds:F1} s"); + Console.WriteLine($" total validated: {report.Elapsed.TotalSeconds:F1} s"); Console.WriteLine( $" peak working set: {report.PeakWorkingSetBytes / 1024.0 / 1024.0:F1} MB"); + Console.WriteLine( + $" peak private: {report.PeakPrivateBytes / 1024.0 / 1024.0:F1} MB"); Console.WriteLine($" output size: {report.OutputBytes / 1024.0 / 1024.0:F1} MB"); Console.WriteLine($" output path: {outputPath}"); } diff --git a/src/AcDream.Bake/EnvCellBakeCatalog.cs b/src/AcDream.Bake/EnvCellBakeCatalog.cs index 5665718b..4e034012 100644 --- a/src/AcDream.Bake/EnvCellBakeCatalog.cs +++ b/src/AcDream.Bake/EnvCellBakeCatalog.cs @@ -53,51 +53,94 @@ public sealed class EnvCellBakeCatalog ArgumentNullException.ThrowIfNull(sources); ArgumentNullException.ThrowIfNull(computeIdentity); - var byGeometry = new Dictionary(); - var fileIds = new HashSet(); - int cellCount = 0; + var builder = new EnvCellBakeCatalogBuilder(computeIdentity); + foreach (var source in sources) + builder.Add(source); + return builder.Build(); + } - foreach (var source in sources.OrderBy(source => source.FileId)) + internal static EnvCellBakeCatalog Create( + IReadOnlyList groups, + int cellCount) => + new(groups, cellCount); +} + +/// +/// Incremental catalog builder used by the full bake so 729,888 EnvCell +/// records and their surface lists are never retained at once. +/// +public sealed class EnvCellBakeCatalogBuilder +{ + private readonly Func, ulong> _computeIdentity; + private readonly Dictionary _byGeometry = new(); + private readonly HashSet _fileIds = new(); + private int _cellCount; + private bool _built; + + public EnvCellBakeCatalogBuilder() + : this(EnvCellGeometryIdentity.Compute) + { + } + + public EnvCellBakeCatalogBuilder( + Func, ulong> computeIdentity) + { + ArgumentNullException.ThrowIfNull(computeIdentity); + _computeIdentity = computeIdentity; + } + + public void Add(EnvCellBakeSource source) + { + if (_built) + throw new InvalidOperationException("the EnvCell bake catalog is already complete"); + + if (!_fileIds.Add(source.FileId)) + throw new InvalidDataException($"duplicate EnvCell file id 0x{source.FileId:X8}"); + + ulong geometryId = _computeIdentity( + source.EnvironmentId, + source.CellStructure, + source.Surfaces); + + if (_byGeometry.TryGetValue(geometryId, out var existing)) { - if (!fileIds.Add(source.FileId)) - throw new InvalidDataException($"duplicate EnvCell file id 0x{source.FileId:X8}"); - - var surfaces = source.Surfaces.ToArray(); - ulong geometryId = computeIdentity( - source.EnvironmentId, - source.CellStructure, - surfaces); - - if (byGeometry.TryGetValue(geometryId, out var existing)) + if (existing.EnvironmentId != source.EnvironmentId || + existing.CellStructure != source.CellStructure || + !SurfacesEqual(existing.Surfaces, source.Surfaces)) { - if (existing.EnvironmentId != source.EnvironmentId || - existing.CellStructure != source.CellStructure || - !existing.Surfaces.AsSpan().SequenceEqual(surfaces)) - { - throw new InvalidDataException( - $"EnvCell geometry identity collision 0x{geometryId:X16}: " + - $"cell 0x{existing.FileIds[0]:X8} and cell 0x{source.FileId:X8} " + - "have different environment/cell-structure/surface tuples"); - } - - existing.FileIds.Add(source.FileId); + throw new InvalidDataException( + $"EnvCell geometry identity collision 0x{geometryId:X16}: " + + $"cell 0x{existing.FileIds[0]:X8} and cell 0x{source.FileId:X8} " + + "have different environment/cell-structure/surface tuples"); } - else - { - byGeometry.Add( + + existing.FileIds.Add(source.FileId); + } + else + { + _byGeometry.Add( + geometryId, + new MutableGroup( geometryId, - new MutableGroup( - geometryId, - source.EnvironmentId, - source.CellStructure, - surfaces, - new List { source.FileId })); - } - - cellCount++; + source.EnvironmentId, + source.CellStructure, + source.Surfaces.ToArray(), + new List { source.FileId })); } - var groups = byGeometry.Values + _cellCount++; + } + + public EnvCellBakeCatalog Build() + { + if (_built) + throw new InvalidOperationException("the EnvCell bake catalog is already complete"); + _built = true; + + foreach (var group in _byGeometry.Values) + group.FileIds.Sort(); + + var groups = _byGeometry.Values .Select(group => new EnvCellGeometryGroup( group.GeometryId, group.EnvironmentId, @@ -107,7 +150,21 @@ public sealed class EnvCellBakeCatalog .OrderBy(group => group.FileIds[0]) .ToArray(); - return new EnvCellBakeCatalog(groups, cellCount); + return EnvCellBakeCatalog.Create(groups, _cellCount); + } + + private static bool SurfacesEqual( + ReadOnlySpan expected, + IReadOnlyList actual) + { + if (expected.Length != actual.Count) + return false; + for (int i = 0; i < expected.Length; i++) + { + if (expected[i] != actual[i]) + return false; + } + return true; } private sealed record MutableGroup( diff --git a/src/AcDream.Content/Pak/PakReader.cs b/src/AcDream.Content/Pak/PakReader.cs index 5c6d262b..c7b01aff 100644 --- a/src/AcDream.Content/Pak/PakReader.cs +++ b/src/AcDream.Content/Pak/PakReader.cs @@ -174,13 +174,13 @@ public sealed class PakReader : IDisposable { } /// - /// Validates the complete immutable artifact before an offline bake - /// publishes it. This is intentionally stricter than runtime lookup: - /// TOC order/ranges and the CRC of every unique physical blob must all be - /// valid. Aliases are checked once per shared receipt. + /// Validates the complete immutable TOC before an offline bake publishes + /// it. This is intentionally stricter than runtime lookup: key order and + /// every physical range must be valid. Blob CRC remains the reader's lazy + /// lookup tripwire; scanning a multi-gigabyte mmap here would fault the + /// whole pak into the process working set just before publication. /// - public void ValidateComplete() { - var validatedReceipts = new HashSet<(ulong Offset, uint Length, uint Crc32)>(); + public void ValidateTocStructure() { ulong previousKey = 0; for (int i = 0; i < _toc.Length; i++) { @@ -198,19 +198,6 @@ public sealed class PakReader : IDisposable { $"(offset={entry.Offset}, length={entry.Length}, " + $"file={_fileLength}, toc@{Header.TocOffset})"); } - - var receipt = (entry.Offset, entry.Length, entry.Crc32); - if (!validatedReceipts.Add(receipt)) - continue; - - var bytes = new byte[entry.Length]; - _accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length); - uint actualCrc = Crc32.Compute(bytes); - if (actualCrc != entry.Crc32) { - throw new InvalidDataException( - $"pak blob 0x{entry.Key:X16} failed CRC validation: " + - $"expected 0x{entry.Crc32:X8}, got 0x{actualCrc:X8}"); - } } } diff --git a/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs b/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs index 5902fd68..f7e5b4bb 100644 --- a/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs +++ b/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.IO; using AcDream.Bake; +using AcDream.Content.Pak; namespace AcDream.Bake.Tests; @@ -76,4 +77,59 @@ public sealed class BakeDeterminismTests : IDisposable { Assert.True(bytesA.AsSpan().SequenceEqual(bytesB), "two bakes of the same id set produced different bytes — the sorted-batching determinism guarantee is broken"); } + + [Fact] + public void Bake_RealDuplicateCells_ShareOneBlobAcrossLandblocksAndThreads() { + var datDir = ResolveDatDir(); + if (datDir is null) return; + + // Full-bake discovery fixture: these eight cells span multiple + // landblocks but have the same complete environment/structure/surface + // tuple. They must remain eight addressable keys backed by one blob. + var ids = new HashSet { + 0x00A90230u, 0x00B901A8u, 0x00E50602u, 0x00E50610u, + 0x00E5061Fu, 0x00F90784u, 0x00F9078Eu, 0x00FA073Eu, + }; + var pathA = NewTempPakPath(); + var pathB = NewTempPakPath(); + + var reportA = BakeRunner.RunDetailed( + new BakeOptions { + DatDir = datDir, + OutPath = pathA, + IdFilter = ids, + Threads = 1, + }); + var reportB = BakeRunner.RunDetailed( + new BakeOptions { + DatDir = datDir, + OutPath = pathB, + IdFilter = ids, + Threads = 8, + }); + + Assert.Equal(8, reportA.EnvCellKeys); + Assert.Equal(1, reportA.UniqueEnvCellGeometries); + Assert.Equal(7, reportA.EnvCellAliases); + Assert.Equal(1, reportA.PhysicalBlobs); + Assert.Equal(reportA.TotalKeys, reportB.TotalKeys); + Assert.True(File.ReadAllBytes(pathA).AsSpan().SequenceEqual(File.ReadAllBytes(pathB))); + + using var reader = new PakReader(pathA); + var receipts = ids + .Select(id => reader.GetTocEntryForTest( + PakKey.Compose(PakAssetType.EnvCellMesh, id))) + .ToArray(); + Assert.All(receipts, receipt => Assert.Equal(receipts[0].Offset, receipt.Offset)); + Assert.All(receipts, receipt => Assert.Equal(receipts[0].Length, receipt.Length)); + Assert.All(receipts, receipt => Assert.Equal(receipts[0].Crc32, receipt.Crc32)); + + Assert.True( + reader.TryReadObjectMeshData( + PakKey.Compose(PakAssetType.EnvCellMesh, ids.Min()), + out var geometry)); + Assert.NotNull(geometry); + Assert.Equal(0xE000_0000_0000_0000UL, + geometry.ObjectId & 0xF000_0000_0000_0000UL); + } } diff --git a/tests/AcDream.Bake.Tests/BakeOutputTransactionTests.cs b/tests/AcDream.Bake.Tests/BakeOutputTransactionTests.cs index 35724ab3..80963cb4 100644 --- a/tests/AcDream.Bake.Tests/BakeOutputTransactionTests.cs +++ b/tests/AcDream.Bake.Tests/BakeOutputTransactionTests.cs @@ -122,7 +122,7 @@ public sealed class BakeOutputTransactionTests : IDisposable } [Fact] - public void ArtifactValidator_RejectsWrongDatIterationAndCorruptBlob() + public void ArtifactValidator_RejectsWrongDatIterationAndCorruptTocRange() { string pak = Path.Combine(_directory, "content.pak"); var expected = Header(); @@ -139,16 +139,15 @@ public sealed class BakeOutputTransactionTests : IDisposable Assert.Throws( () => BakeArtifactValidator.Validate(pak, wrongIteration, expectedTocCount: 1)); - long offset; - using (var reader = new PakReader(pak)) - offset = reader.GetBlobOffsetForTest( - PakKey.Compose(PakAssetType.GfxObjMesh, 1)); using (var stream = new FileStream(pak, FileMode.Open, FileAccess.ReadWrite)) { - stream.Position = offset; - int value = stream.ReadByte(); - stream.Position = offset; - stream.WriteByte((byte)(value ^ 0xFF)); + var header = PakHeader.ReadFrom(stream); + stream.Position = checked((long)header.TocOffset + 16); + Span invalidLength = stackalloc byte[4]; + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian( + invalidLength, + uint.MaxValue); + stream.Write(invalidLength); } Assert.Throws(