feat(bake): extract each EnvCell geometry once
This commit is contained in:
Erik
parent
0dee14765b
commit
90b378cc70
4 changed files with 582 additions and 153 deletions
|
|
@ -1,98 +1,148 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Numerics;
|
||||
using AcDream.Content;
|
||||
using AcDream.Content.Pak;
|
||||
using DatReaderWriter;
|
||||
using DatReaderWriter.DBObjs;
|
||||
using DatReaderWriter.Options;
|
||||
using DatEnvironment = DatReaderWriter.DBObjs.Environment;
|
||||
|
||||
namespace AcDream.Bake;
|
||||
|
||||
/// <summary>Options for one bake run (parsed from CLI args by Program, or built directly by tests).</summary>
|
||||
public sealed record BakeOptions {
|
||||
public sealed record BakeOptions
|
||||
{
|
||||
public required string DatDir { get; init; }
|
||||
public required string OutPath { get; init; }
|
||||
public HashSet<uint>? IdFilter { get; init; }
|
||||
public HashSet<uint>? LandblockFilter { get; init; }
|
||||
public int Threads { get; init; } = System.Environment.ProcessorCount;
|
||||
public CancellationToken CancellationToken { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>Compact result used by the full-scale gate and deterministic tests.</summary>
|
||||
public sealed record BakeReport
|
||||
{
|
||||
public required int GfxObjKeys { get; init; }
|
||||
public required int SetupKeys { get; init; }
|
||||
public required int EnvCellKeys { get; init; }
|
||||
public required int UniqueEnvCellGeometries { get; init; }
|
||||
public required int EnvCellAliases { get; init; }
|
||||
public required int SideStagedKeys { get; init; }
|
||||
public required int PhysicalBlobs { get; init; }
|
||||
public required int TotalKeys { get; init; }
|
||||
public required int Failures { get; init; }
|
||||
public required TimeSpan Elapsed { get; init; }
|
||||
public required long OutputBytes { get; init; }
|
||||
public required long PeakWorkingSetBytes { get; init; }
|
||||
|
||||
public double EnvCellDedupRatio =>
|
||||
UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The bake pipeline: enumerate ids -> sorted-batch parallel extraction ->
|
||||
/// deterministic pak write. Public (rather than inline in Program) so the
|
||||
/// dat-gated byte-reproducibility test can drive the REAL pipeline.
|
||||
///
|
||||
/// <para><b>Determinism + bounded memory (review finding 1):</b> the full id
|
||||
/// list is built first and sorted by pak key; work proceeds in fixed-size
|
||||
/// batches — extraction is parallel WITHIN a batch, then the batch's results
|
||||
/// are sorted by key and written sequentially before the next batch starts.
|
||||
/// Batches are contiguous key ranges, so the blob region ends up in global
|
||||
/// key order regardless of thread scheduling, and memory is bounded by one
|
||||
/// batch's decoded output instead of the whole game's. Side-staged
|
||||
/// particle-preload meshes are deduped into a keyed map as each batch
|
||||
/// drains (first instance wins — extraction output per id is
|
||||
/// deterministic, so instance choice cannot affect bytes) and written after
|
||||
/// all batches, sorted by key, skipping keys already written.</para>
|
||||
/// Offline content pipeline: enumerate source IDs, catalog EnvCell geometry by
|
||||
/// the same content identity used at runtime, extract each unique payload once,
|
||||
/// and stream deterministic blobs plus ordinary TOC aliases.
|
||||
/// </summary>
|
||||
public static class BakeRunner {
|
||||
/// <summary>Ids extracted in parallel per batch before the sequential sorted write. Bounds peak memory (~512 decoded ObjectMeshData) while keeping the workers busy.</summary>
|
||||
public static class BakeRunner
|
||||
{
|
||||
/// <summary>
|
||||
/// Items extracted in parallel per batch before sequential sorted write.
|
||||
/// Bounds peak decoded output while keeping workers busy.
|
||||
/// </summary>
|
||||
private const int BatchSize = 512;
|
||||
|
||||
public static int Run(BakeOptions options) {
|
||||
public static int Run(BakeOptions options)
|
||||
{
|
||||
RunDetailed(options);
|
||||
return 0;
|
||||
}
|
||||
|
||||
public static BakeReport RunDetailed(BakeOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
if (options.Threads <= 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(options), "thread count must be positive");
|
||||
|
||||
var cancellationToken = options.CancellationToken;
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
Console.WriteLine("acdream-bake");
|
||||
Console.WriteLine($"dat dir: {options.DatDir}");
|
||||
Console.WriteLine($"out: {options.OutPath}");
|
||||
Console.WriteLine($"threads: {options.Threads}");
|
||||
Console.WriteLine();
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var stopwatch = Stopwatch.StartNew();
|
||||
using var dats = new DatCollection(options.DatDir, DatAccessType.Read);
|
||||
// The unified AcDream.Content.DatCollectionAdapter (MP1b review
|
||||
// finding 7): same instance class the client and the equivalence
|
||||
// suite use — adapter drift between bake and runtime is impossible
|
||||
// by construction.
|
||||
using var datReaderWriter = new DatCollectionAdapter(dats);
|
||||
var extractorLogger = new ConsoleErrorLogger(nameof(MeshExtractor));
|
||||
|
||||
// Thread-safe side-stage sink for particle-preload meshes MeshExtractor
|
||||
// emits mid-extraction (MP1a documented contract; extractor is shared
|
||||
// across the batch workers).
|
||||
var sideStaged = new ConcurrentQueue<ObjectMeshData>();
|
||||
var extractor = new MeshExtractor(datReaderWriter, extractorLogger, data => sideStaged.Enqueue(data));
|
||||
var extractor = new MeshExtractor(
|
||||
datReaderWriter,
|
||||
extractorLogger,
|
||||
data => sideStaged.Enqueue(data));
|
||||
|
||||
// ---- enumeration -----------------------------------------------------
|
||||
var failures = new ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)>();
|
||||
|
||||
var gfxObjIds = dats.GetAllIdsOfType<GfxObj>().ToList();
|
||||
var setupIds = dats.GetAllIdsOfType<Setup>().ToList();
|
||||
// ---- enumeration and content-identity catalog ----------------------
|
||||
|
||||
var gfxObjIds = dats.GetAllIdsOfType<GfxObj>().OrderBy(id => id).ToList();
|
||||
var setupIds = dats.GetAllIdsOfType<Setup>().OrderBy(id => id).ToList();
|
||||
var envCellIds = EnumerateEnvCellIds(dats, options.LandblockFilter);
|
||||
|
||||
if (options.IdFilter is not null) {
|
||||
if (options.IdFilter is not null)
|
||||
{
|
||||
gfxObjIds = gfxObjIds.Where(options.IdFilter.Contains).ToList();
|
||||
setupIds = setupIds.Where(options.IdFilter.Contains).ToList();
|
||||
envCellIds = envCellIds.Where(options.IdFilter.Contains).ToList();
|
||||
}
|
||||
|
||||
// Full work list sorted by pak key — the batching below preserves this
|
||||
// global order on disk (determinism precondition).
|
||||
var work = new List<(ulong Key, PakAssetType Type, uint FileId)>(gfxObjIds.Count + setupIds.Count + envCellIds.Count);
|
||||
work.AddRange(gfxObjIds.Select(id => (PakKey.Compose(PakAssetType.GfxObjMesh, id), PakAssetType.GfxObjMesh, id)));
|
||||
work.AddRange(setupIds.Select(id => (PakKey.Compose(PakAssetType.SetupMesh, id), PakAssetType.SetupMesh, id)));
|
||||
work.AddRange(envCellIds.Select(id => (PakKey.Compose(PakAssetType.EnvCellMesh, id), PakAssetType.EnvCellMesh, id)));
|
||||
work.Sort((a, b) => a.Key.CompareTo(b.Key));
|
||||
var envCellSources = new List<EnvCellBakeSource>(envCellIds.Count);
|
||||
foreach (uint fileId in envCellIds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (!datReaderWriter.Cell.TryGet<EnvCell>(fileId, out var envCell) || envCell is null)
|
||||
{
|
||||
failures.Add((PakAssetType.EnvCellMesh, fileId, "EnvCell DAT record was not found"));
|
||||
continue;
|
||||
}
|
||||
|
||||
Console.WriteLine($"enumerated: {gfxObjIds.Count:N0} GfxObj, {setupIds.Count:N0} Setup, {envCellIds.Count:N0} EnvCell " +
|
||||
$"({work.Count:N0} total)");
|
||||
envCellSources.Add(
|
||||
new EnvCellBakeSource(
|
||||
fileId,
|
||||
envCell.EnvironmentId,
|
||||
envCell.CellStructure,
|
||||
envCell.Surfaces));
|
||||
}
|
||||
|
||||
var envCatalog = EnvCellBakeCatalog.Build(envCellSources);
|
||||
var ordinaryWork = new List<(ulong Key, PakAssetType Type, uint FileId)>(
|
||||
gfxObjIds.Count + setupIds.Count);
|
||||
ordinaryWork.AddRange(
|
||||
gfxObjIds.Select(
|
||||
id => (PakKey.Compose(PakAssetType.GfxObjMesh, id), PakAssetType.GfxObjMesh, id)));
|
||||
ordinaryWork.AddRange(
|
||||
setupIds.Select(
|
||||
id => (PakKey.Compose(PakAssetType.SetupMesh, id), PakAssetType.SetupMesh, id)));
|
||||
ordinaryWork.Sort((a, b) => a.Key.CompareTo(b.Key));
|
||||
|
||||
Console.WriteLine(
|
||||
$"enumerated: {gfxObjIds.Count:N0} GfxObj, {setupIds.Count:N0} Setup, " +
|
||||
$"{envCellIds.Count:N0} EnvCell");
|
||||
Console.WriteLine(
|
||||
$"EnvCell catalog: {envCatalog.CellCount:N0} valid cells -> " +
|
||||
$"{envCatalog.UniqueGeometryCount:N0} unique geometries + " +
|
||||
$"{envCatalog.AliasCount:N0} aliases " +
|
||||
$"({(envCatalog.UniqueGeometryCount == 0 ? 0 : (double)envCatalog.CellCount / envCatalog.UniqueGeometryCount):F1}x)");
|
||||
Console.WriteLine();
|
||||
|
||||
// ---- sorted-batch extraction + streaming write -------------------------
|
||||
// ---- sorted-batch extraction and deterministic write ---------------
|
||||
|
||||
var header = new PakHeader {
|
||||
var header = new PakHeader
|
||||
{
|
||||
PortalIteration = (uint)dats.Portal.Iteration!.CurrentIteration,
|
||||
CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration,
|
||||
HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration,
|
||||
|
|
@ -100,153 +150,328 @@ public static class BakeRunner {
|
|||
BakeToolVersion = 1,
|
||||
};
|
||||
|
||||
var failures = new ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)>();
|
||||
var writtenKeys = new HashSet<ulong>();
|
||||
var sideStagedByKey = new Dictionary<ulong, ObjectMeshData>();
|
||||
long completed = 0;
|
||||
int writtenCounts0 = 0, writtenCounts1 = 0, writtenCounts2 = 0; // GfxObj, Setup, EnvCell
|
||||
int totalExtractionJobs = ordinaryWork.Count + envCatalog.UniqueGeometryCount;
|
||||
int gfxObjWritten = 0;
|
||||
int setupWritten = 0;
|
||||
int envCellKeysWritten = 0;
|
||||
int envCellGeometriesWritten = 0;
|
||||
var lastProgressReport = Stopwatch.StartNew();
|
||||
|
||||
using (var writer = new PakWriter(options.OutPath, header)) {
|
||||
for (int batchStart = 0; batchStart < work.Count; batchStart += BatchSize) {
|
||||
var batch = work.Skip(batchStart).Take(BatchSize).ToList();
|
||||
var batchResults = new ConcurrentBag<(ulong Key, PakAssetType Type, ObjectMeshData Data)>();
|
||||
using (var writer = new PakWriter(options.OutPath, header))
|
||||
{
|
||||
for (int batchStart = 0; batchStart < ordinaryWork.Count; batchStart += BatchSize)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var batch = ordinaryWork.Skip(batchStart).Take(BatchSize).ToArray();
|
||||
var batchResults =
|
||||
new ConcurrentBag<(ulong Key, PakAssetType Type, ObjectMeshData Data)>();
|
||||
|
||||
Parallel.ForEach(
|
||||
batch,
|
||||
new ParallelOptions { MaxDegreeOfParallelism = options.Threads },
|
||||
item => {
|
||||
new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = options.Threads,
|
||||
CancellationToken = cancellationToken,
|
||||
},
|
||||
item =>
|
||||
{
|
||||
var (key, type, fileId) = item;
|
||||
try {
|
||||
// EnvCell entries store the cell's synthetic geometry mesh
|
||||
// (bit 32 set — see MeshExtractor.PrepareMeshData's EnvCell
|
||||
// branch); the cell's static objects are covered by their
|
||||
// own GfxObj/Setup entries.
|
||||
ulong extractorId = type == PakAssetType.EnvCellMesh ? fileId | 0x1_0000_0000UL : fileId;
|
||||
// isSetup: matches the runtime's own request sites —
|
||||
// WbMeshAdapter.IncrementRefCount/EnsureLoaded pass
|
||||
// isSetup: false for every MeshRef id (incl. cell-geometry
|
||||
// ids); only true Setup extraction passes true. (Review
|
||||
// finding 10 — keep aligned with PakEquivalenceTests.)
|
||||
try
|
||||
{
|
||||
bool isSetup = type == PakAssetType.SetupMesh;
|
||||
var data = extractor.PrepareMeshData(extractorId, isSetup);
|
||||
if (data is not null) {
|
||||
var data = extractor.PrepareMeshData(fileId, isSetup, cancellationToken);
|
||||
if (data is not null)
|
||||
batchResults.Add((key, type, data));
|
||||
}
|
||||
else {
|
||||
failures.Add((type, fileId, "extractor returned null (no polygons or unresolvable id)"));
|
||||
}
|
||||
else
|
||||
failures.Add((type, fileId, "extractor returned null"));
|
||||
}
|
||||
catch (Exception ex) {
|
||||
// A malformed dat entry skips that id — never fatal to the
|
||||
// bake, matching the runtime's own per-id behavior.
|
||||
failures.Add((type, fileId, ex.Message));
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
failures.Add((type, fileId, exception.Message));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Increment(ref completed);
|
||||
}
|
||||
|
||||
Interlocked.Increment(ref completed);
|
||||
});
|
||||
|
||||
// Sequential sorted write: batch results in key order.
|
||||
foreach (var (key, type, data) in batchResults.OrderBy(r => r.Key)) {
|
||||
foreach (var (key, type, data) in batchResults.OrderBy(result => result.Key))
|
||||
{
|
||||
writer.AddBlob(key, data);
|
||||
writtenKeys.Add(key);
|
||||
switch (type) {
|
||||
case PakAssetType.GfxObjMesh: writtenCounts0++; break;
|
||||
case PakAssetType.SetupMesh: writtenCounts1++; break;
|
||||
case PakAssetType.EnvCellMesh: writtenCounts2++; break;
|
||||
if (type == PakAssetType.GfxObjMesh)
|
||||
gfxObjWritten++;
|
||||
else
|
||||
setupWritten++;
|
||||
}
|
||||
|
||||
DrainSideStaged(sideStaged, sideStagedByKey);
|
||||
ReportProgressIfDue(
|
||||
completed,
|
||||
totalExtractionJobs,
|
||||
failures.Count,
|
||||
stopwatch.Elapsed,
|
||||
lastProgressReport,
|
||||
batchStart + BatchSize >= ordinaryWork.Count &&
|
||||
envCatalog.UniqueGeometryCount == 0);
|
||||
}
|
||||
|
||||
for (int batchStart = 0; batchStart < envCatalog.Groups.Count; batchStart += BatchSize)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var batch = envCatalog.Groups.Skip(batchStart).Take(BatchSize).ToArray();
|
||||
var batchResults =
|
||||
new ConcurrentBag<(EnvCellGeometryGroup Group, ObjectMeshData Data)>();
|
||||
|
||||
Parallel.ForEach(
|
||||
batch,
|
||||
new ParallelOptions
|
||||
{
|
||||
MaxDegreeOfParallelism = options.Threads,
|
||||
CancellationToken = cancellationToken,
|
||||
},
|
||||
group =>
|
||||
{
|
||||
try
|
||||
{
|
||||
uint environmentDid = 0x0D00_0000u | group.EnvironmentId;
|
||||
if (!datReaderWriter.Portal.TryGet<DatEnvironment>(
|
||||
environmentDid,
|
||||
out var environment) ||
|
||||
environment is null ||
|
||||
!environment.Cells.TryGetValue(
|
||||
group.CellStructure,
|
||||
out var cellStruct))
|
||||
{
|
||||
failures.Add((
|
||||
PakAssetType.EnvCellMesh,
|
||||
group.FileIds[0],
|
||||
$"environment 0x{environmentDid:X8} cell structure " +
|
||||
$"{group.CellStructure} was not found"));
|
||||
return;
|
||||
}
|
||||
|
||||
var data = extractor.PrepareCellStructMeshData(
|
||||
group.GeometryId,
|
||||
cellStruct,
|
||||
group.Surfaces,
|
||||
Matrix4x4.Identity,
|
||||
cancellationToken);
|
||||
if (data is not null)
|
||||
batchResults.Add((group, data));
|
||||
else
|
||||
failures.Add((
|
||||
PakAssetType.EnvCellMesh,
|
||||
group.FileIds[0],
|
||||
"unique cell-structure extraction returned null"));
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
failures.Add((
|
||||
PakAssetType.EnvCellMesh,
|
||||
group.FileIds[0],
|
||||
exception.Message));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Interlocked.Increment(ref completed);
|
||||
}
|
||||
});
|
||||
|
||||
foreach (var (group, data) in batchResults.OrderBy(result => result.Group.FileIds[0]))
|
||||
{
|
||||
ulong primaryKey =
|
||||
PakKey.Compose(PakAssetType.EnvCellMesh, group.FileIds[0]);
|
||||
writer.AddBlob(primaryKey, data);
|
||||
writtenKeys.Add(primaryKey);
|
||||
envCellKeysWritten++;
|
||||
envCellGeometriesWritten++;
|
||||
|
||||
for (int i = 1; i < group.FileIds.Length; i++)
|
||||
{
|
||||
ulong aliasKey =
|
||||
PakKey.Compose(PakAssetType.EnvCellMesh, group.FileIds[i]);
|
||||
writer.AddAlias(aliasKey, primaryKey);
|
||||
writtenKeys.Add(aliasKey);
|
||||
envCellKeysWritten++;
|
||||
}
|
||||
}
|
||||
|
||||
// Drain side-staged preloads per batch into the dedup map so the
|
||||
// queue never grows past one batch's emissions (memory bound);
|
||||
// written after all batches, sorted (see below).
|
||||
while (sideStaged.TryDequeue(out var staged)) {
|
||||
uint fileId = (uint)(staged.ObjectId & 0xFFFFFFFFu);
|
||||
ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId);
|
||||
if (!sideStagedByKey.ContainsKey(key)) sideStagedByKey[key] = staged;
|
||||
}
|
||||
|
||||
if (lastProgressReport.Elapsed.TotalSeconds >= 5 || batchStart + BatchSize >= work.Count) {
|
||||
ReportProgress(completed, work.Count, failures.Count, sw.Elapsed);
|
||||
lastProgressReport.Restart();
|
||||
}
|
||||
DrainSideStaged(sideStaged, sideStagedByKey);
|
||||
ReportProgressIfDue(
|
||||
completed,
|
||||
totalExtractionJobs,
|
||||
failures.Count,
|
||||
stopwatch.Elapsed,
|
||||
lastProgressReport,
|
||||
batchStart + BatchSize >= envCatalog.Groups.Count);
|
||||
}
|
||||
|
||||
// Side-staged preload meshes not already covered by a primary entry:
|
||||
// sorted by key for deterministic placement.
|
||||
int sideStagedWritten = 0, sideStagedDuped = 0;
|
||||
foreach (var (key, data) in sideStagedByKey.OrderBy(kv => kv.Key)) {
|
||||
if (writtenKeys.Contains(key)) { sideStagedDuped++; continue; }
|
||||
int sideStagedWritten = 0;
|
||||
int sideStagedDuped = 0;
|
||||
foreach (var (key, data) in sideStagedByKey.OrderBy(pair => pair.Key))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (writtenKeys.Contains(key))
|
||||
{
|
||||
sideStagedDuped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
writer.AddBlob(key, data);
|
||||
writtenKeys.Add(key);
|
||||
sideStagedWritten++;
|
||||
}
|
||||
|
||||
writer.Finish();
|
||||
sw.Stop();
|
||||
stopwatch.Stop();
|
||||
|
||||
var outSize = new FileInfo(options.OutPath).Length;
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("=== bake summary ===");
|
||||
Console.WriteLine($" GfxObj baked: {writtenCounts0:N0}");
|
||||
Console.WriteLine($" Setup baked: {writtenCounts1:N0}");
|
||||
Console.WriteLine($" EnvCell baked: {writtenCounts2:N0}");
|
||||
Console.WriteLine($" side-staged baked (particle preload GfxObjs): {sideStagedWritten:N0} ({sideStagedDuped:N0} deduped)");
|
||||
Console.WriteLine($" total blobs: {writtenKeys.Count:N0}");
|
||||
Console.WriteLine($" failures: {failures.Count:N0}");
|
||||
Console.WriteLine($" elapsed: {sw.Elapsed.TotalSeconds:F1} s");
|
||||
Console.WriteLine($" output size: {outSize / 1024.0 / 1024.0:F1} MB");
|
||||
Console.WriteLine($" output path: {options.OutPath}");
|
||||
var report = new BakeReport
|
||||
{
|
||||
GfxObjKeys = gfxObjWritten,
|
||||
SetupKeys = setupWritten,
|
||||
EnvCellKeys = envCellKeysWritten,
|
||||
UniqueEnvCellGeometries = envCellGeometriesWritten,
|
||||
EnvCellAliases = envCellKeysWritten - envCellGeometriesWritten,
|
||||
SideStagedKeys = sideStagedWritten,
|
||||
PhysicalBlobs =
|
||||
gfxObjWritten + setupWritten + envCellGeometriesWritten + sideStagedWritten,
|
||||
TotalKeys = writtenKeys.Count,
|
||||
Failures = failures.Count,
|
||||
Elapsed = stopwatch.Elapsed,
|
||||
OutputBytes = new FileInfo(options.OutPath).Length,
|
||||
PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
|
||||
};
|
||||
|
||||
PrintSummary(report, sideStagedDuped, options.OutPath);
|
||||
PrintFailures(failures);
|
||||
return report;
|
||||
}
|
||||
|
||||
if (!failures.IsEmpty) {
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"failures ({failures.Count}):");
|
||||
foreach (var (type, fileId, reason) in failures.OrderBy(f => f.FileId).Take(200)) {
|
||||
Console.WriteLine($" {type,-12} 0x{fileId:X8}: {reason}");
|
||||
}
|
||||
if (failures.Count > 200) Console.WriteLine($" ... and {failures.Count - 200} more");
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
private static void ReportProgress(long done, int total, int failures, TimeSpan elapsed) {
|
||||
private static void DrainSideStaged(
|
||||
ConcurrentQueue<ObjectMeshData> sideStaged,
|
||||
Dictionary<ulong, ObjectMeshData> sideStagedByKey)
|
||||
{
|
||||
while (sideStaged.TryDequeue(out var staged))
|
||||
{
|
||||
uint fileId = (uint)(staged.ObjectId & 0xFFFF_FFFFu);
|
||||
ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId);
|
||||
sideStagedByKey.TryAdd(key, staged);
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReportProgressIfDue(
|
||||
long done,
|
||||
int total,
|
||||
int failures,
|
||||
TimeSpan elapsed,
|
||||
Stopwatch lastProgressReport,
|
||||
bool final)
|
||||
{
|
||||
if (!final && lastProgressReport.Elapsed.TotalSeconds < 5)
|
||||
return;
|
||||
|
||||
double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0;
|
||||
double etaSeconds = rate > 0 ? (total - done) / rate : 0;
|
||||
Console.WriteLine($"[{elapsed:hh\\:mm\\:ss}] baked {done:N0}/{total:N0}, failures={failures:N0}, " +
|
||||
$"elapsed={elapsed.TotalSeconds:F0}s, ETA={etaSeconds:F0}s");
|
||||
Console.WriteLine(
|
||||
$"[{elapsed:hh\\:mm\\:ss}] extracted {done:N0}/{total:N0}, " +
|
||||
$"failures={failures:N0}, elapsed={elapsed.TotalSeconds:F0}s, " +
|
||||
$"ETA={etaSeconds:F0}s");
|
||||
lastProgressReport.Restart();
|
||||
}
|
||||
|
||||
private static void PrintSummary(BakeReport report, int sideStagedDuped, string outputPath)
|
||||
{
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("=== bake summary ===");
|
||||
Console.WriteLine($" GfxObj keys: {report.GfxObjKeys:N0}");
|
||||
Console.WriteLine($" Setup keys: {report.SetupKeys:N0}");
|
||||
Console.WriteLine($" EnvCell keys: {report.EnvCellKeys:N0}");
|
||||
Console.WriteLine($" EnvCell unique: {report.UniqueEnvCellGeometries:N0}");
|
||||
Console.WriteLine($" EnvCell aliases: {report.EnvCellAliases:N0}");
|
||||
Console.WriteLine($" EnvCell dedup: {report.EnvCellDedupRatio:F1}x");
|
||||
Console.WriteLine(
|
||||
$" side-staged keys: {report.SideStagedKeys:N0} " +
|
||||
$"({sideStagedDuped:N0} duplicate keys)");
|
||||
Console.WriteLine($" physical blobs: {report.PhysicalBlobs:N0}");
|
||||
Console.WriteLine($" total keys: {report.TotalKeys:N0}");
|
||||
Console.WriteLine($" failures: {report.Failures:N0}");
|
||||
Console.WriteLine($" elapsed: {report.Elapsed.TotalSeconds:F1} s");
|
||||
Console.WriteLine(
|
||||
$" peak working set: {report.PeakWorkingSetBytes / 1024.0 / 1024.0:F1} MB");
|
||||
Console.WriteLine($" output size: {report.OutputBytes / 1024.0 / 1024.0:F1} MB");
|
||||
Console.WriteLine($" output path: {outputPath}");
|
||||
}
|
||||
|
||||
private static void PrintFailures(
|
||||
ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)> failures)
|
||||
{
|
||||
if (failures.IsEmpty)
|
||||
return;
|
||||
|
||||
Console.WriteLine();
|
||||
Console.WriteLine($"failures ({failures.Count}):");
|
||||
foreach (var (type, fileId, reason) in failures
|
||||
.OrderBy(failure => failure.Type)
|
||||
.ThenBy(failure => failure.FileId)
|
||||
.Take(200))
|
||||
{
|
||||
Console.WriteLine($" {type,-12} 0x{fileId:X8}: {reason}");
|
||||
}
|
||||
|
||||
if (failures.Count > 200)
|
||||
Console.WriteLine($" ... and {failures.Count - 200} more");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Enumerates EnvCell ids by walking the cell dat's LandBlockInfo entries
|
||||
/// (0xFFFE low 16 bits) and, for each, the NumCells-derived cell id range —
|
||||
/// GetAllIdsOfType<T>() does not cover cell-dat range-based types
|
||||
/// (DatReaderWriter's documented limitation; see the low-16-bit bucketing
|
||||
/// idiom in src/AcDream.Cli/Program.cs's CountCellByLow16, and the
|
||||
/// firstCellId/NumCells hydration idiom in GameWindow.BuildPhysicsDatBundle
|
||||
/// / BuildInteriorEntitiesForStreaming).
|
||||
/// Enumerates EnvCell IDs by walking each cell DAT LandBlockInfo record's
|
||||
/// NumCells range. DatReaderWriter cannot enumerate these range-shaped IDs
|
||||
/// through GetAllIdsOfType.
|
||||
/// </summary>
|
||||
private static List<uint> EnumerateEnvCellIds(DatCollection dats, HashSet<uint>? landblockFilter) {
|
||||
private static List<uint> EnumerateEnvCellIds(
|
||||
DatCollection dats,
|
||||
HashSet<uint>? landblockFilter)
|
||||
{
|
||||
var landblockInfoIds = new List<uint>();
|
||||
foreach (var file in dats.Cell.Tree) {
|
||||
if ((file.Id & 0xFFFFu) != 0xFFFEu) continue;
|
||||
uint landblockId = file.Id & 0xFFFF0000u;
|
||||
if (landblockFilter is not null && !landblockFilter.Contains(landblockId)) continue;
|
||||
foreach (var file in dats.Cell.Tree)
|
||||
{
|
||||
if ((file.Id & 0xFFFFu) != 0xFFFEu)
|
||||
continue;
|
||||
|
||||
uint landblockId = file.Id & 0xFFFF_0000u;
|
||||
if (landblockFilter is not null && !landblockFilter.Contains(landblockId))
|
||||
continue;
|
||||
landblockInfoIds.Add(file.Id);
|
||||
}
|
||||
|
||||
landblockInfoIds.Sort();
|
||||
var envCellIds = new List<uint>();
|
||||
foreach (var lbInfoId in landblockInfoIds) {
|
||||
if (!dats.Cell.TryGet<LandBlockInfo>(lbInfoId, out var lbInfo) || lbInfo is null) continue;
|
||||
if (lbInfo.NumCells == 0) continue;
|
||||
|
||||
uint landblockId = lbInfoId & 0xFFFF0000u;
|
||||
uint firstCellId = landblockId | 0x0100u;
|
||||
for (uint offset = 0; offset < lbInfo.NumCells; offset++) {
|
||||
envCellIds.Add(firstCellId + offset);
|
||||
foreach (uint landblockInfoId in landblockInfoIds)
|
||||
{
|
||||
if (!dats.Cell.TryGet<LandBlockInfo>(landblockInfoId, out var info) ||
|
||||
info is null ||
|
||||
info.NumCells == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
uint firstCellId = (landblockInfoId & 0xFFFF_0000u) | 0x0100u;
|
||||
for (uint offset = 0; offset < info.NumCells; offset++)
|
||||
envCellIds.Add(firstCellId + offset);
|
||||
}
|
||||
|
||||
return envCellIds;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
119
src/AcDream.Bake/EnvCellBakeCatalog.cs
Normal file
119
src/AcDream.Bake/EnvCellBakeCatalog.cs
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
using AcDream.Core.Rendering.Wb;
|
||||
|
||||
namespace AcDream.Bake;
|
||||
|
||||
/// <summary>
|
||||
/// The identity-bearing portion of one EnvCell DAT record needed by the bake.
|
||||
/// Position, portals, and static objects are instance data and intentionally do
|
||||
/// not participate in shared shell geometry.
|
||||
/// </summary>
|
||||
public sealed record EnvCellBakeSource(
|
||||
uint FileId,
|
||||
uint EnvironmentId,
|
||||
ushort CellStructure,
|
||||
IReadOnlyList<ushort> Surfaces);
|
||||
|
||||
/// <summary>One unique shell geometry and every independently addressable cell key that uses it.</summary>
|
||||
public sealed record EnvCellGeometryGroup(
|
||||
ulong GeometryId,
|
||||
uint EnvironmentId,
|
||||
ushort CellStructure,
|
||||
ushort[] Surfaces,
|
||||
uint[] FileIds);
|
||||
|
||||
/// <summary>
|
||||
/// Builds the deterministic file-id alias map used by the offline bake.
|
||||
/// Geometry equality is checked against the complete source tuple so a hash
|
||||
/// collision can never silently merge different cells.
|
||||
/// </summary>
|
||||
public sealed class EnvCellBakeCatalog
|
||||
{
|
||||
public IReadOnlyList<EnvCellGeometryGroup> Groups { get; }
|
||||
public int CellCount { get; }
|
||||
public int UniqueGeometryCount => Groups.Count;
|
||||
public int AliasCount => CellCount - UniqueGeometryCount;
|
||||
|
||||
private EnvCellBakeCatalog(IReadOnlyList<EnvCellGeometryGroup> groups, int cellCount)
|
||||
{
|
||||
Groups = groups;
|
||||
CellCount = cellCount;
|
||||
}
|
||||
|
||||
public static EnvCellBakeCatalog Build(IEnumerable<EnvCellBakeSource> sources) =>
|
||||
Build(sources, EnvCellGeometryIdentity.Compute);
|
||||
|
||||
/// <summary>
|
||||
/// Identity injection exists for collision-path conformance tests. Production
|
||||
/// always calls the overload bound to <see cref="EnvCellGeometryIdentity"/>.
|
||||
/// </summary>
|
||||
public static EnvCellBakeCatalog Build(
|
||||
IEnumerable<EnvCellBakeSource> sources,
|
||||
Func<uint, ushort, IReadOnlyList<ushort>, ulong> computeIdentity)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sources);
|
||||
ArgumentNullException.ThrowIfNull(computeIdentity);
|
||||
|
||||
var byGeometry = new Dictionary<ulong, MutableGroup>();
|
||||
var fileIds = new HashSet<uint>();
|
||||
int cellCount = 0;
|
||||
|
||||
foreach (var source in sources.OrderBy(source => source.FileId))
|
||||
{
|
||||
if (!fileIds.Add(source.FileId))
|
||||
throw new InvalidDataException($"duplicate EnvCell file id 0x{source.FileId:X8}");
|
||||
|
||||
var surfaces = source.Surfaces.ToArray();
|
||||
ulong geometryId = computeIdentity(
|
||||
source.EnvironmentId,
|
||||
source.CellStructure,
|
||||
surfaces);
|
||||
|
||||
if (byGeometry.TryGetValue(geometryId, out var existing))
|
||||
{
|
||||
if (existing.EnvironmentId != source.EnvironmentId ||
|
||||
existing.CellStructure != source.CellStructure ||
|
||||
!existing.Surfaces.AsSpan().SequenceEqual(surfaces))
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"EnvCell geometry identity collision 0x{geometryId:X16}: " +
|
||||
$"cell 0x{existing.FileIds[0]:X8} and cell 0x{source.FileId:X8} " +
|
||||
"have different environment/cell-structure/surface tuples");
|
||||
}
|
||||
|
||||
existing.FileIds.Add(source.FileId);
|
||||
}
|
||||
else
|
||||
{
|
||||
byGeometry.Add(
|
||||
geometryId,
|
||||
new MutableGroup(
|
||||
geometryId,
|
||||
source.EnvironmentId,
|
||||
source.CellStructure,
|
||||
surfaces,
|
||||
new List<uint> { source.FileId }));
|
||||
}
|
||||
|
||||
cellCount++;
|
||||
}
|
||||
|
||||
var groups = byGeometry.Values
|
||||
.Select(group => new EnvCellGeometryGroup(
|
||||
group.GeometryId,
|
||||
group.EnvironmentId,
|
||||
group.CellStructure,
|
||||
group.Surfaces,
|
||||
group.FileIds.ToArray()))
|
||||
.OrderBy(group => group.FileIds[0])
|
||||
.ToArray();
|
||||
|
||||
return new EnvCellBakeCatalog(groups, cellCount);
|
||||
}
|
||||
|
||||
private sealed record MutableGroup(
|
||||
ulong GeometryId,
|
||||
uint EnvironmentId,
|
||||
ushort CellStructure,
|
||||
ushort[] Surfaces,
|
||||
List<uint> FileIds);
|
||||
}
|
||||
|
|
@ -686,7 +686,7 @@ public sealed class MeshExtractor {
|
|||
};
|
||||
}
|
||||
|
||||
public ObjectMeshData? PrepareCellStructMeshData(ulong id, CellStruct cellStruct, List<ushort> surfaceOverrides, Matrix4x4 transform, CancellationToken ct) {
|
||||
public ObjectMeshData? PrepareCellStructMeshData(ulong id, CellStruct cellStruct, IReadOnlyList<ushort> surfaceOverrides, Matrix4x4 transform, CancellationToken ct) {
|
||||
var vertices = new List<VertexPositionNormalTexture>();
|
||||
var UVLookup = new Dictionary<(ushort vertId, ushort uvIdx, bool isNeg), ushort>();
|
||||
var batchesByFormat = new Dictionary<(int Width, int Height, TextureFormat Format), List<TextureBatchData>>();
|
||||
|
|
|
|||
85
tests/AcDream.Bake.Tests/EnvCellBakeCatalogTests.cs
Normal file
85
tests/AcDream.Bake.Tests/EnvCellBakeCatalogTests.cs
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
namespace AcDream.Bake.Tests;
|
||||
|
||||
public sealed class EnvCellBakeCatalogTests
|
||||
{
|
||||
[Fact]
|
||||
public void Build_GroupsEqualGeometryAndPreservesEverySortedFileId()
|
||||
{
|
||||
var catalog = EnvCellBakeCatalog.Build(
|
||||
[
|
||||
new(0x2222_0102, 7, 3, new ushort[] { 10, 20 }),
|
||||
new(0x1111_0100, 7, 3, new ushort[] { 10, 20 }),
|
||||
new(0x1111_0101, 7, 4, new ushort[] { 10, 20 }),
|
||||
new(0x2222_0100, 7, 3, new ushort[] { 10, 20 }),
|
||||
]);
|
||||
|
||||
Assert.Equal(4, catalog.CellCount);
|
||||
Assert.Equal(2, catalog.UniqueGeometryCount);
|
||||
Assert.Equal(2, catalog.AliasCount);
|
||||
Assert.Equal(
|
||||
new uint[] { 0x1111_0100, 0x2222_0100, 0x2222_0102 },
|
||||
catalog.Groups[0].FileIds);
|
||||
Assert.Equal(new uint[] { 0x1111_0101 }, catalog.Groups[1].FileIds);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Build_IsDeterministicAcrossSourceOrder()
|
||||
{
|
||||
EnvCellBakeSource[] sources =
|
||||
[
|
||||
new(4, 9, 2, new ushort[] { 5, 6 }),
|
||||
new(2, 9, 2, new ushort[] { 5, 6 }),
|
||||
new(3, 1, 8, new ushort[] { 7 }),
|
||||
];
|
||||
|
||||
var forward = EnvCellBakeCatalog.Build(sources);
|
||||
var reverse = EnvCellBakeCatalog.Build(sources.Reverse());
|
||||
|
||||
Assert.Equal(forward.Groups, reverse.Groups, GroupComparer.Instance);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Build_RejectsIdentityCollisionWithDifferentTuple()
|
||||
{
|
||||
EnvCellBakeSource[] sources =
|
||||
[
|
||||
new(1, 10, 20, new ushort[] { 30 }),
|
||||
new(2, 11, 20, new ushort[] { 30 }),
|
||||
];
|
||||
|
||||
var exception = Assert.Throws<InvalidDataException>(
|
||||
() => EnvCellBakeCatalog.Build(sources, (_, _, _) => 0x2_0000_0000));
|
||||
|
||||
Assert.Contains("collision", exception.Message);
|
||||
Assert.Contains("0x00000001", exception.Message);
|
||||
Assert.Contains("0x00000002", exception.Message);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Build_RejectsDuplicateFileId()
|
||||
{
|
||||
EnvCellBakeSource[] sources =
|
||||
[
|
||||
new(1, 10, 20, Array.Empty<ushort>()),
|
||||
new(1, 10, 20, Array.Empty<ushort>()),
|
||||
];
|
||||
|
||||
Assert.Throws<InvalidDataException>(() => EnvCellBakeCatalog.Build(sources));
|
||||
}
|
||||
|
||||
private sealed class GroupComparer : IEqualityComparer<EnvCellGeometryGroup>
|
||||
{
|
||||
public static GroupComparer Instance { get; } = new();
|
||||
|
||||
public bool Equals(EnvCellGeometryGroup? x, EnvCellGeometryGroup? y) =>
|
||||
x is not null &&
|
||||
y is not null &&
|
||||
x.GeometryId == y.GeometryId &&
|
||||
x.EnvironmentId == y.EnvironmentId &&
|
||||
x.CellStructure == y.CellStructure &&
|
||||
x.Surfaces.AsSpan().SequenceEqual(y.Surfaces) &&
|
||||
x.FileIds.AsSpan().SequenceEqual(y.FileIds);
|
||||
|
||||
public int GetHashCode(EnvCellGeometryGroup obj) => obj.GeometryId.GetHashCode();
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue