erik/acdream
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

perf(bake): stream catalog and bound validation residency

Browse source
This commit is contained in:
Erik 2026-07-24 14:05:30 +02:00
parent b7b9aaa9dd
commit 90bf6bbf45
7 changed files with 217 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

56
src/AcDream.Bake/BakeRunner.cs
View file

@ -35,9 +35,11 @@ public sealed record BakeReport
public required int PhysicalBlobs { get; init; }
public required int TotalKeys { get; init; }
public required int Failures { get; init; }
public required TimeSpan ExtractionAndWriteElapsed { get; init; }
public required TimeSpan Elapsed { get; init; }
public required long OutputBytes { get; init; }
public required long PeakWorkingSetBytes { get; init; }
public required long PeakPrivateBytes { get; init; }
public double EnvCellDedupRatio =>
UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries;
@ -54,7 +56,7 @@ public static class BakeRunner
/// Items extracted in parallel per batch before sequential sorted write.
/// Bounds peak decoded output while keeping workers busy.
/// </summary>
private const int BatchSize = 512;
private const int BatchSize = 16;
public static int Run(BakeOptions options)
{
@ -69,6 +71,7 @@ public static class BakeRunner
throw new ArgumentOutOfRangeException(nameof(options), "thread count must be positive");
options.CancellationToken.ThrowIfCancellationRequested();
var totalStopwatch = Stopwatch.StartNew();
var report = BakeOutputTransaction.WriteValidateAndPublish(
options.OutPath,
temporaryPath => RunCore(
@ -79,6 +82,15 @@ public static class BakeRunner
result.Header,
result.TotalKeys),
options.CancellationToken);
totalStopwatch.Stop();
report = report with
{
Elapsed = totalStopwatch.Elapsed,
OutputBytes = new FileInfo(options.OutPath).Length,
PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64,
};
PrintSummary(report, options.OutPath);
return report;
@ -119,7 +131,7 @@ public static class BakeRunner
envCellIds = envCellIds.Where(options.IdFilter.Contains).ToList();
}
var envCellSources = new List<EnvCellBakeSource>(envCellIds.Count);
var envCatalogBuilder = new EnvCellBakeCatalogBuilder();
foreach (uint fileId in envCellIds)
{
cancellationToken.ThrowIfCancellationRequested();
@ -129,7 +141,7 @@ public static class BakeRunner
continue;
}
envCellSources.Add(
envCatalogBuilder.Add(
new EnvCellBakeSource(
fileId,
envCell.EnvironmentId,
@ -137,7 +149,7 @@ public static class BakeRunner
envCell.Surfaces));
}
var envCatalog = EnvCellBakeCatalog.Build(envCellSources);
var envCatalog = envCatalogBuilder.Build();
var ordinaryWork = new List<(ulong Key, PakAssetType Type, uint FileId)>(
gfxObjIds.Count + setupIds.Count);
ordinaryWork.AddRange(
@ -177,6 +189,7 @@ public static class BakeRunner
int setupWritten = 0;
int envCellKeysWritten = 0;
int envCellGeometriesWritten = 0;
int sideStagedDuped = 0;
var lastProgressReport = Stopwatch.StartNew();
using (var writer = new PakWriter(options.OutPath, header))
@ -231,7 +244,10 @@ public static class BakeRunner
setupWritten++;
}
DrainSideStaged(sideStaged, sideStagedByKey);
sideStagedDuped += DrainSideStaged(
sideStaged,
sideStagedByKey,
writtenKeys);
ReportProgressIfDue(
completed,
totalExtractionJobs,
@ -327,7 +343,10 @@ public static class BakeRunner
}
}
DrainSideStaged(sideStaged, sideStagedByKey);
sideStagedDuped += DrainSideStaged(
sideStaged,
sideStagedByKey,
writtenKeys);
ReportProgressIfDue(
completed,
totalExtractionJobs,
@ -338,7 +357,6 @@ public static class BakeRunner
}
int sideStagedWritten = 0;
int sideStagedDuped = 0;
foreach (var (key, data) in sideStagedByKey.OrderBy(pair => pair.Key))
{
cancellationToken.ThrowIfCancellationRequested();
@ -369,9 +387,11 @@ public static class BakeRunner
gfxObjWritten + setupWritten + envCellGeometriesWritten + sideStagedWritten,
TotalKeys = writtenKeys.Count,
Failures = failures.Count,
ExtractionAndWriteElapsed = stopwatch.Elapsed,
Elapsed = stopwatch.Elapsed,
OutputBytes = new FileInfo(options.OutPath).Length,
PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64,
};
report = report with { SideStagedDuplicateKeys = sideStagedDuped };
@ -380,16 +400,20 @@ public static class BakeRunner
}
}
private static void DrainSideStaged(
private static int DrainSideStaged(
ConcurrentQueue<ObjectMeshData> sideStaged,
Dictionary<ulong, ObjectMeshData> sideStagedByKey)
Dictionary<ulong, ObjectMeshData> sideStagedByKey,
HashSet<ulong> writtenKeys)
{
int duplicates = 0;
while (sideStaged.TryDequeue(out var staged))
{
uint fileId = (uint)(staged.ObjectId & 0xFFFF_FFFFu);
ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId);
sideStagedByKey.TryAdd(key, staged);
if (writtenKeys.Contains(key) || !sideStagedByKey.TryAdd(key, staged))
duplicates++;
}
return duplicates;
}
private static void ReportProgressIfDue(
@ -405,10 +429,14 @@ public static class BakeRunner
double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0;
double etaSeconds = rate > 0 ? (total - done) / rate : 0;
using var process = Process.GetCurrentProcess();
process.Refresh();
long managedHeap = GC.GetGCMemoryInfo().HeapSizeBytes;
Console.WriteLine(
$"[{elapsed:hh\\:mm\\:ss}] extracted {done:N0}/{total:N0}, " +
$"failures={failures:N0}, elapsed={elapsed.TotalSeconds:F0}s, " +
$"ETA={etaSeconds:F0}s");
$"ETA={etaSeconds:F0}s, private={process.PrivateMemorySize64 / 1024.0 / 1024.0:F0}MB, " +
$"managed={managedHeap / 1024.0 / 1024.0:F0}MB");
lastProgressReport.Restart();
}
@ -428,9 +456,13 @@ public static class BakeRunner
Console.WriteLine($" physical blobs: {report.PhysicalBlobs:N0}");
Console.WriteLine($" total keys: {report.TotalKeys:N0}");
Console.WriteLine($" failures: {report.Failures:N0}");
Console.WriteLine($" elapsed: {report.Elapsed.TotalSeconds:F1} s");
Console.WriteLine(
$" extract + write: {report.ExtractionAndWriteElapsed.TotalSeconds:F1} s");
Console.WriteLine($" total validated: {report.Elapsed.TotalSeconds:F1} s");
Console.WriteLine(
$" peak working set: {report.PeakWorkingSetBytes / 1024.0 / 1024.0:F1} MB");
Console.WriteLine(
$" peak private: {report.PeakPrivateBytes / 1024.0 / 1024.0:F1} MB");
Console.WriteLine($" output size: {report.OutputBytes / 1024.0 / 1024.0:F1} MB");
Console.WriteLine($" output path: {outputPath}");
}