erik/acdream
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 2

perf(bake): stream catalog and bound validation residency

Browse source
This commit is contained in:
Erik 2026-07-24 14:05:30 +02:00
parent b7b9aaa9dd
commit 90bf6bbf45
7 changed files with 217 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/AcDream.Bake/BakeArtifactValidator.cs
View file

@ -34,6 +34,6 @@ public static class BakeArtifactValidator
$"bake TOC count {actual.TocCount} does not match expected " +
$"{expectedTocCount}");
reader.ValidateComplete();
reader.ValidateTocStructure();
}
}

10
src/AcDream.Bake/BakeOutputTransaction.cs
View file

@ -37,10 +37,12 @@ public static class BakeOutputTransaction
validateTemporary(temporaryPath, result);
cancellationToken.ThrowIfCancellationRequested();
if (File.Exists(fullDestination))
File.Replace(temporaryPath, fullDestination, null, ignoreMetadataErrors: true);
else
File.Move(temporaryPath, fullDestination);
// Same-volume MoveFileEx/rename is the publication primitive.
// File.Replace additionally performs destination metadata/backup
// semantics and proved brittle for a validated 28 GiB artifact on
// Windows. Move(overwrite:true) changes only the directory entry:
// readers observe the complete old file or the complete new file.
File.Move(temporaryPath, fullDestination, overwrite: true);
return result;
}

56
src/AcDream.Bake/BakeRunner.cs
View file

@ -35,9 +35,11 @@ public sealed record BakeReport
public required int PhysicalBlobs { get; init; }
public required int TotalKeys { get; init; }
public required int Failures { get; init; }
public required TimeSpan ExtractionAndWriteElapsed { get; init; }
public required TimeSpan Elapsed { get; init; }
public required long OutputBytes { get; init; }
public required long PeakWorkingSetBytes { get; init; }
public required long PeakPrivateBytes { get; init; }
public double EnvCellDedupRatio =>
UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries;
@ -54,7 +56,7 @@ public static class BakeRunner
/// Items extracted in parallel per batch before sequential sorted write.
/// Bounds peak decoded output while keeping workers busy.
/// </summary>
private const int BatchSize = 512;
private const int BatchSize = 16;
public static int Run(BakeOptions options)
{
@ -69,6 +71,7 @@ public static class BakeRunner
throw new ArgumentOutOfRangeException(nameof(options), "thread count must be positive");
options.CancellationToken.ThrowIfCancellationRequested();
var totalStopwatch = Stopwatch.StartNew();
var report = BakeOutputTransaction.WriteValidateAndPublish(
options.OutPath,
temporaryPath => RunCore(
@ -79,6 +82,15 @@ public static class BakeRunner
result.Header,
result.TotalKeys),
options.CancellationToken);
totalStopwatch.Stop();
report = report with
{
Elapsed = totalStopwatch.Elapsed,
OutputBytes = new FileInfo(options.OutPath).Length,
PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64,
};
PrintSummary(report, options.OutPath);
return report;
@ -119,7 +131,7 @@ public static class BakeRunner
envCellIds = envCellIds.Where(options.IdFilter.Contains).ToList();
}
var envCellSources = new List<EnvCellBakeSource>(envCellIds.Count);
var envCatalogBuilder = new EnvCellBakeCatalogBuilder();
foreach (uint fileId in envCellIds)
{
cancellationToken.ThrowIfCancellationRequested();
@ -129,7 +141,7 @@ public static class BakeRunner
continue;
}
envCellSources.Add(
envCatalogBuilder.Add(
new EnvCellBakeSource(
fileId,
envCell.EnvironmentId,
@ -137,7 +149,7 @@ public static class BakeRunner
envCell.Surfaces));
}
var envCatalog = EnvCellBakeCatalog.Build(envCellSources);
var envCatalog = envCatalogBuilder.Build();
var ordinaryWork = new List<(ulong Key, PakAssetType Type, uint FileId)>(
gfxObjIds.Count + setupIds.Count);
ordinaryWork.AddRange(
@ -177,6 +189,7 @@ public static class BakeRunner
int setupWritten = 0;
int envCellKeysWritten = 0;
int envCellGeometriesWritten = 0;
int sideStagedDuped = 0;
var lastProgressReport = Stopwatch.StartNew();
using (var writer = new PakWriter(options.OutPath, header))
@ -231,7 +244,10 @@ public static class BakeRunner
setupWritten++;
}
DrainSideStaged(sideStaged, sideStagedByKey);
sideStagedDuped += DrainSideStaged(
sideStaged,
sideStagedByKey,
writtenKeys);
ReportProgressIfDue(
completed,
totalExtractionJobs,
@ -327,7 +343,10 @@ public static class BakeRunner
}
}
DrainSideStaged(sideStaged, sideStagedByKey);
sideStagedDuped += DrainSideStaged(
sideStaged,
sideStagedByKey,
writtenKeys);
ReportProgressIfDue(
completed,
totalExtractionJobs,
@ -338,7 +357,6 @@ public static class BakeRunner
}
int sideStagedWritten = 0;
int sideStagedDuped = 0;
foreach (var (key, data) in sideStagedByKey.OrderBy(pair => pair.Key))
{
cancellationToken.ThrowIfCancellationRequested();
@ -369,9 +387,11 @@ public static class BakeRunner
gfxObjWritten + setupWritten + envCellGeometriesWritten + sideStagedWritten,
TotalKeys = writtenKeys.Count,
Failures = failures.Count,
ExtractionAndWriteElapsed = stopwatch.Elapsed,
Elapsed = stopwatch.Elapsed,
OutputBytes = new FileInfo(options.OutPath).Length,
PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64,
};
report = report with { SideStagedDuplicateKeys = sideStagedDuped };
@ -380,16 +400,20 @@ public static class BakeRunner
}
}
private static void DrainSideStaged(
private static int DrainSideStaged(
ConcurrentQueue<ObjectMeshData> sideStaged,
Dictionary<ulong, ObjectMeshData> sideStagedByKey)
Dictionary<ulong, ObjectMeshData> sideStagedByKey,
HashSet<ulong> writtenKeys)
{
int duplicates = 0;
while (sideStaged.TryDequeue(out var staged))
{
uint fileId = (uint)(staged.ObjectId & 0xFFFF_FFFFu);
ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId);
sideStagedByKey.TryAdd(key, staged);
if (writtenKeys.Contains(key) || !sideStagedByKey.TryAdd(key, staged))
duplicates++;
}
return duplicates;
}
private static void ReportProgressIfDue(
@ -405,10 +429,14 @@ public static class BakeRunner
double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0;
double etaSeconds = rate > 0 ? (total - done) / rate : 0;
using var process = Process.GetCurrentProcess();
process.Refresh();
long managedHeap = GC.GetGCMemoryInfo().HeapSizeBytes;
Console.WriteLine(
$"[{elapsed:hh\\:mm\\:ss}] extracted {done:N0}/{total:N0}, " +
$"failures={failures:N0}, elapsed={elapsed.TotalSeconds:F0}s, " +
$"ETA={etaSeconds:F0}s");
$"ETA={etaSeconds:F0}s, private={process.PrivateMemorySize64 / 1024.0 / 1024.0:F0}MB, " +
$"managed={managedHeap / 1024.0 / 1024.0:F0}MB");
lastProgressReport.Restart();
}
@ -428,9 +456,13 @@ public static class BakeRunner
Console.WriteLine($" physical blobs: {report.PhysicalBlobs:N0}");
Console.WriteLine($" total keys: {report.TotalKeys:N0}");
Console.WriteLine($" failures: {report.Failures:N0}");
Console.WriteLine($" elapsed: {report.Elapsed.TotalSeconds:F1} s");
Console.WriteLine(
$" extract + write: {report.ExtractionAndWriteElapsed.TotalSeconds:F1} s");
Console.WriteLine($" total validated: {report.Elapsed.TotalSeconds:F1} s");
Console.WriteLine(
$" peak working set: {report.PeakWorkingSetBytes / 1024.0 / 1024.0:F1} MB");
Console.WriteLine(
$" peak private: {report.PeakPrivateBytes / 1024.0 / 1024.0:F1} MB");
Console.WriteLine($" output size: {report.OutputBytes / 1024.0 / 1024.0:F1} MB");
Console.WriteLine($" output path: {outputPath}");
}

135
src/AcDream.Bake/EnvCellBakeCatalog.cs
View file

@ -53,51 +53,94 @@ public sealed class EnvCellBakeCatalog
ArgumentNullException.ThrowIfNull(sources);
ArgumentNullException.ThrowIfNull(computeIdentity);
var byGeometry = new Dictionary<ulong, MutableGroup>();
var fileIds = new HashSet<uint>();
int cellCount = 0;
var builder = new EnvCellBakeCatalogBuilder(computeIdentity);
foreach (var source in sources)
builder.Add(source);
return builder.Build();
}
foreach (var source in sources.OrderBy(source => source.FileId))
internal static EnvCellBakeCatalog Create(
IReadOnlyList<EnvCellGeometryGroup> groups,
int cellCount) =>
new(groups, cellCount);
}
/// <summary>
/// Incremental catalog builder used by the full bake so 729,888 EnvCell
/// records and their surface lists are never retained at once.
/// </summary>
public sealed class EnvCellBakeCatalogBuilder
{
private readonly Func<uint, ushort, IReadOnlyList<ushort>, ulong> _computeIdentity;
private readonly Dictionary<ulong, MutableGroup> _byGeometry = new();
private readonly HashSet<uint> _fileIds = new();
private int _cellCount;
private bool _built;
public EnvCellBakeCatalogBuilder()
: this(EnvCellGeometryIdentity.Compute)
{
}
public EnvCellBakeCatalogBuilder(
Func<uint, ushort, IReadOnlyList<ushort>, ulong> computeIdentity)
{
ArgumentNullException.ThrowIfNull(computeIdentity);
_computeIdentity = computeIdentity;
}
public void Add(EnvCellBakeSource source)
{
if (_built)
throw new InvalidOperationException("the EnvCell bake catalog is already complete");
if (!_fileIds.Add(source.FileId))
throw new InvalidDataException($"duplicate EnvCell file id 0x{source.FileId:X8}");
ulong geometryId = _computeIdentity(
source.EnvironmentId,
source.CellStructure,
source.Surfaces);
if (_byGeometry.TryGetValue(geometryId, out var existing))
{
if (!fileIds.Add(source.FileId))
throw new InvalidDataException($"duplicate EnvCell file id 0x{source.FileId:X8}");
var surfaces = source.Surfaces.ToArray();
ulong geometryId = computeIdentity(
source.EnvironmentId,
source.CellStructure,
surfaces);
if (byGeometry.TryGetValue(geometryId, out var existing))
if (existing.EnvironmentId != source.EnvironmentId ||
existing.CellStructure != source.CellStructure ||
!SurfacesEqual(existing.Surfaces, source.Surfaces))
{
if (existing.EnvironmentId != source.EnvironmentId ||
existing.CellStructure != source.CellStructure ||
!existing.Surfaces.AsSpan().SequenceEqual(surfaces))
{
throw new InvalidDataException(
$"EnvCell geometry identity collision 0x{geometryId:X16}: " +
$"cell 0x{existing.FileIds[0]:X8} and cell 0x{source.FileId:X8} " +
"have different environment/cell-structure/surface tuples");
}
existing.FileIds.Add(source.FileId);
throw new InvalidDataException(
$"EnvCell geometry identity collision 0x{geometryId:X16}: " +
$"cell 0x{existing.FileIds[0]:X8} and cell 0x{source.FileId:X8} " +
"have different environment/cell-structure/surface tuples");
}
else
{
byGeometry.Add(
existing.FileIds.Add(source.FileId);
}
else
{
_byGeometry.Add(
geometryId,
new MutableGroup(
geometryId,
new MutableGroup(
geometryId,
source.EnvironmentId,
source.CellStructure,
surfaces,
new List<uint> { source.FileId }));
}
cellCount++;
source.EnvironmentId,
source.CellStructure,
source.Surfaces.ToArray(),
new List<uint> { source.FileId }));
}
var groups = byGeometry.Values
_cellCount++;
}
public EnvCellBakeCatalog Build()
{
if (_built)
throw new InvalidOperationException("the EnvCell bake catalog is already complete");
_built = true;
foreach (var group in _byGeometry.Values)
group.FileIds.Sort();
var groups = _byGeometry.Values
.Select(group => new EnvCellGeometryGroup(
group.GeometryId,
group.EnvironmentId,
@ -107,7 +150,21 @@ public sealed class EnvCellBakeCatalog
.OrderBy(group => group.FileIds[0])
.ToArray();
return new EnvCellBakeCatalog(groups, cellCount);
return EnvCellBakeCatalog.Create(groups, _cellCount);
}
private static bool SurfacesEqual(
ReadOnlySpan<ushort> expected,
IReadOnlyList<ushort> actual)
{
if (expected.Length != actual.Count)
return false;
for (int i = 0; i < expected.Length; i++)
{
if (expected[i] != actual[i])
return false;
}
return true;
}
private sealed record MutableGroup(

25
src/AcDream.Content/Pak/PakReader.cs
View file

@ -174,13 +174,13 @@ public sealed class PakReader : IDisposable {
}
/// <summary>
/// Validates the complete immutable artifact before an offline bake
/// publishes it. This is intentionally stricter than runtime lookup:
/// TOC order/ranges and the CRC of every unique physical blob must all be
/// valid. Aliases are checked once per shared receipt.
/// Validates the complete immutable TOC before an offline bake publishes
/// it. This is intentionally stricter than runtime lookup: key order and
/// every physical range must be valid. Blob CRC remains the reader's lazy
/// lookup tripwire; scanning a multi-gigabyte mmap here would fault the
/// whole pak into the process working set just before publication.
/// </summary>
public void ValidateComplete() {
var validatedReceipts = new HashSet<(ulong Offset, uint Length, uint Crc32)>();
public void ValidateTocStructure() {
ulong previousKey = 0;
for (int i = 0; i < _toc.Length; i++) {
@ -198,19 +198,6 @@ public sealed class PakReader : IDisposable {
$"(offset={entry.Offset}, length={entry.Length}, " +
$"file={_fileLength}, toc@{Header.TocOffset})");
}
var receipt = (entry.Offset, entry.Length, entry.Crc32);
if (!validatedReceipts.Add(receipt))
continue;
var bytes = new byte[entry.Length];
_accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length);
uint actualCrc = Crc32.Compute(bytes);
if (actualCrc != entry.Crc32) {
throw new InvalidDataException(
$"pak blob 0x{entry.Key:X16} failed CRC validation: " +
$"expected 0x{entry.Crc32:X8}, got 0x{actualCrc:X8}");
}
}
}