erik/acdream
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

feat(content): bake and read flat collision assets

Browse source 
Append strict collision/topology payloads to the existing prepared package so later physics cutover can drop parsed DAT graphs without adding a second mapping or changing traversal behavior. The full 2,232,170-key catalog is deterministic across worker counts, exact-byte aliased, corruption-isolated, and cancellation-safe.
This commit is contained in:
Erik 2026-07-25 15:22:08 +02:00
parent d9300c7854
commit 9cd42417a8
29 changed files with 2868 additions and 63 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/AcDream.Bake/AcDream.Bake.csproj
View file

@ -10,6 +10,10 @@
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<InternalsVisibleTo Include="AcDream.Bake.Tests" />
</ItemGroup>
<ItemGroup>
<!-- NO Silk.NET / GL packages here — ever. acdream-bake is an offline CLI
that drives the GL-free AcDream.Content.MeshExtractor; it must not

16
src/AcDream.Bake/BakeArtifactValidator.cs
View file

@ -8,7 +8,8 @@ public static class BakeArtifactValidator
public static void Validate(
string path,
in PakHeader expectedHeader,
int expectedTocCount)
int expectedTocCount,
IReadOnlyDictionary<PakAssetType, int>? expectedTypeCounts = null)
{
using var reader = new PakReader(path);
var actual = reader.Header;
@ -35,5 +36,18 @@ public static class BakeArtifactValidator
$"{expectedTocCount}");
reader.ValidateTocStructure();
if (expectedTypeCounts is null)
return;
foreach ((PakAssetType type, int expectedCount) in expectedTypeCounts)
{
int actualCount = reader.CountEntries(type);
if (actualCount != expectedCount)
{
throw new InvalidDataException(
$"bake catalog type {type} contains {actualCount} keys; " +
$"expected {expectedCount}");
}
}
}
}

517
src/AcDream.Bake/BakeRunner.cs
View file

@ -3,9 +3,12 @@ using System.Diagnostics;
using System.Numerics;
using AcDream.Content;
using AcDream.Content.Pak;
using AcDream.Core.Physics;
using DatReaderWriter;
using DatReaderWriter.DBObjs;
using DatReaderWriter.Enums;
using DatReaderWriter.Options;
using DatReaderWriter.Types;
using DatEnvironment = DatReaderWriter.DBObjs.Environment;
namespace AcDream.Bake;
@ -30,6 +33,16 @@ public sealed record BakeReport
public required int EnvCellKeys { get; init; }
public required int UniqueEnvCellGeometries { get; init; }
public required int EnvCellAliases { get; init; }
public required int GfxObjCollisionKeys { get; init; }
public required int UniqueGfxObjCollisions { get; init; }
public required int GfxObjCollisionAliases { get; init; }
public required int SetupCollisionKeys { get; init; }
public required int UniqueSetupCollisions { get; init; }
public required int SetupCollisionAliases { get; init; }
public required int CellStructureCollisionKeys { get; init; }
public required int UniqueCellStructureCollisions { get; init; }
public required int CellStructureCollisionAliases { get; init; }
public required int EnvCellTopologyKeys { get; init; }
public required int SideStagedKeys { get; init; }
public int SideStagedDuplicateKeys { get; init; }
public required int PhysicalBlobs { get; init; }
@ -40,6 +53,11 @@ public sealed record BakeReport
public required long OutputBytes { get; init; }
public required long PeakWorkingSetBytes { get; init; }
public required long PeakPrivateBytes { get; init; }
public required IReadOnlyDictionary<PakAssetType, int> TypeCounts
{
get;
init;
}
public double EnvCellDedupRatio =>
UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries;
@ -80,7 +98,8 @@ public static class BakeRunner
(temporaryPath, result) => BakeArtifactValidator.Validate(
temporaryPath,
result.Header,
result.TotalKeys),
result.TotalKeys,
result.TypeCounts),
options.CancellationToken);
totalStopwatch.Stop();
@ -371,9 +390,402 @@ public static class BakeRunner
sideStagedWritten++;
}
// ---- immutable flat collision payloads ------------------------
uint[] collisionGfxIds = gfxObjIds
.Concat(
sideStagedByKey.Keys.Select(
key => PakKey.Decompose(key).FileId))
.Distinct()
.Order()
.ToArray();
var collisionWork =
new List<(ulong Key, PakAssetType Type, uint FileId)>(
collisionGfxIds.Length + setupIds.Count);
collisionWork.AddRange(
collisionGfxIds.Select(
id => (
PakKey.Compose(PakAssetType.GfxObjCollision, id),
PakAssetType.GfxObjCollision,
id)));
collisionWork.AddRange(
setupIds.Select(
id => (
PakKey.Compose(PakAssetType.SetupCollision, id),
PakAssetType.SetupCollision,
id)));
collisionWork.Sort((left, right) => left.Key.CompareTo(right.Key));
var gfxCollisionAliases = new ExactPayloadAliasCatalog();
var setupCollisionAliases = new ExactPayloadAliasCatalog();
var cellStructureAliases = new ExactPayloadAliasCatalog();
int gfxCollisionWritten = 0;
int uniqueGfxCollisions = 0;
int gfxCollisionAliasCount = 0;
int setupCollisionWritten = 0;
int uniqueSetupCollisions = 0;
int setupCollisionAliasCount = 0;
int cellStructureWritten = 0;
int uniqueCellStructures = 0;
int cellStructureAliasCount = 0;
int envCellTopologyWritten = 0;
long collisionCompleted = 0;
int totalCollisionJobs =
collisionWork.Count
+ envCatalog.UniqueGeometryCount
+ envCatalog.CellCount;
var collisionStopwatch = Stopwatch.StartNew();
lastProgressReport.Restart();
Console.WriteLine();
Console.WriteLine(
$"collision catalog: {collisionGfxIds.Length:N0} GfxObj, " +
$"{setupIds.Count:N0} Setup, " +
$"{envCatalog.UniqueGeometryCount:N0} CellStruct source groups, " +
$"{envCatalog.CellCount:N0} EnvCell topology records");
for (int batchStart = 0;
batchStart < collisionWork.Count;
batchStart += BatchSize)
{
cancellationToken.ThrowIfCancellationRequested();
var batch = collisionWork
.Skip(batchStart)
.Take(BatchSize)
.ToArray();
var batchResults =
new ConcurrentBag<CollisionPayloadResult>();
Parallel.ForEach(
batch,
new ParallelOptions
{
MaxDegreeOfParallelism = options.Threads,
CancellationToken = cancellationToken,
},
item =>
{
var (key, type, fileId) = item;
try
{
byte[] payload;
if (type == PakAssetType.GfxObjCollision)
{
if (!datReaderWriter.Portal.TryGet<GfxObj>(
fileId,
out GfxObj? gfxObj)
|| gfxObj is null)
{
failures.Add((
type,
fileId,
"GfxObj DAT record was not found"));
return;
}
FlatGfxObjCollisionAsset asset =
FlatCollisionAssetBuilder.FlattenGfxObj(
gfxObj);
payload = FlatCollisionAssetSerializer.Serialize(
asset,
cancellationToken);
}
else
{
if (!datReaderWriter.Portal.TryGet<Setup>(
fileId,
out Setup? setup)
|| setup is null)
{
failures.Add((
type,
fileId,
"Setup DAT record was not found"));
return;
}
FlatSetupCollision asset =
FlatCollisionAssetBuilder.FlattenSetup(
setup);
payload = FlatCollisionAssetSerializer.Serialize(
asset,
cancellationToken);
}
batchResults.Add(
new CollisionPayloadResult(
key,
type,
fileId,
payload));
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
failures.Add((type, fileId, exception.Message));
}
finally
{
Interlocked.Increment(ref collisionCompleted);
}
});
foreach (CollisionPayloadResult result in
batchResults.OrderBy(result => result.Key))
{
ExactPayloadAliasCatalog aliases =
result.Type == PakAssetType.GfxObjCollision
? gfxCollisionAliases
: setupCollisionAliases;
bool alias = WriteExactPayload(
writer,
result.Key,
result.Payload,
aliases);
writtenKeys.Add(result.Key);
if (result.Type == PakAssetType.GfxObjCollision)
{
gfxCollisionWritten++;
if (alias)
gfxCollisionAliasCount++;
else
uniqueGfxCollisions++;
}
else
{
setupCollisionWritten++;
if (alias)
setupCollisionAliasCount++;
else
uniqueSetupCollisions++;
}
}
ReportProgressIfDue(
collisionCompleted,
totalCollisionJobs,
failures.Count,
collisionStopwatch.Elapsed,
lastProgressReport,
final: false);
}
for (int groupBatchStart = 0;
groupBatchStart < envCatalog.Groups.Count;
groupBatchStart += BatchSize)
{
cancellationToken.ThrowIfCancellationRequested();
EnvCellGeometryGroup[] groupBatch = envCatalog.Groups
.Skip(groupBatchStart)
.Take(BatchSize)
.ToArray();
var structureResults =
new ConcurrentBag<CellStructurePayloadResult>();
Parallel.ForEach(
groupBatch,
new ParallelOptions
{
MaxDegreeOfParallelism = options.Threads,
CancellationToken = cancellationToken,
},
group =>
{
try
{
uint environmentDid =
0x0D00_0000u | group.EnvironmentId;
if (!datReaderWriter.Portal.TryGet<DatEnvironment>(
environmentDid,
out DatEnvironment? environment)
|| environment is null
|| !environment.Cells.TryGetValue(
group.CellStructure,
out CellStruct? cellStruct)
|| cellStruct is null)
{
failures.Add((
PakAssetType.CellStructureCollision,
group.FileIds[0],
$"environment 0x{environmentDid:X8} cell " +
$"structure {group.CellStructure} was not found"));
Interlocked.Add(
ref collisionCompleted,
group.FileIds.Length);
return;
}
FlatCellStructureCollisionAsset structure =
FlatCollisionAssetBuilder.FlattenCellStructure(
cellStruct);
byte[] payload =
FlatCollisionAssetSerializer.Serialize(
structure,
cancellationToken);
structureResults.Add(
new CellStructurePayloadResult(
group,
structure,
payload));
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
failures.Add((
PakAssetType.CellStructureCollision,
group.FileIds[0],
exception.Message));
Interlocked.Add(
ref collisionCompleted,
group.FileIds.Length);
}
finally
{
Interlocked.Increment(ref collisionCompleted);
}
});
foreach (CellStructurePayloadResult result in
structureResults.OrderBy(
result => result.Group.FileIds[0]))
{
(int physical, int aliases) = WriteExactPayloadGroup(
writer,
PakAssetType.CellStructureCollision,
result.Group.FileIds,
result.Payload,
cellStructureAliases,
writtenKeys);
cellStructureWritten += result.Group.FileIds.Length;
uniqueCellStructures += physical;
cellStructureAliasCount += aliases;
for (int topologyBatchStart = 0;
topologyBatchStart < result.Group.FileIds.Length;
topologyBatchStart += BatchSize)
{
cancellationToken.ThrowIfCancellationRequested();
uint[] topologyBatch = result.Group.FileIds
.Skip(topologyBatchStart)
.Take(BatchSize)
.ToArray();
var topologyResults =
new ConcurrentBag<CollisionPayloadResult>();
Parallel.ForEach(
topologyBatch,
new ParallelOptions
{
MaxDegreeOfParallelism = options.Threads,
CancellationToken = cancellationToken,
},
fileId =>
{
try
{
if (!datReaderWriter.Cell.TryGet<EnvCell>(
fileId,
out EnvCell? envCell)
|| envCell is null)
{
failures.Add((
PakAssetType.EnvCellTopology,
fileId,
"EnvCell DAT record was not found"));
return;
}
FlatEnvCellTopology topology =
FlatCollisionAssetBuilder
.FlattenEnvCellTopology(
fileId,
envCell,
result.Structure
.PortalPolygons);
byte[] payload =
FlatCollisionAssetSerializer.Serialize(
topology,
cancellationToken);
topologyResults.Add(
new CollisionPayloadResult(
PakKey.Compose(
PakAssetType.EnvCellTopology,
fileId),
PakAssetType.EnvCellTopology,
fileId,
payload));
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
failures.Add((
PakAssetType.EnvCellTopology,
fileId,
exception.Message));
}
finally
{
Interlocked.Increment(
ref collisionCompleted);
}
});
foreach (CollisionPayloadResult topology in
topologyResults.OrderBy(
result => result.Key))
{
writer.AddBlob(
topology.Key,
topology.Payload);
writtenKeys.Add(topology.Key);
envCellTopologyWritten++;
}
ReportProgressIfDue(
collisionCompleted,
totalCollisionJobs,
failures.Count,
collisionStopwatch.Elapsed,
lastProgressReport,
final: false);
}
}
}
collisionStopwatch.Stop();
ReportProgressIfDue(
collisionCompleted,
totalCollisionJobs,
failures.Count,
collisionStopwatch.Elapsed,
lastProgressReport,
final: true);
writer.Finish();
stopwatch.Stop();
var typeCounts = new Dictionary<PakAssetType, int>
{
[PakAssetType.GfxObjMesh] =
gfxObjWritten + sideStagedWritten,
[PakAssetType.SetupMesh] = setupWritten,
[PakAssetType.EnvCellMesh] = envCellKeysWritten,
[PakAssetType.GfxObjCollision] = gfxCollisionWritten,
[PakAssetType.SetupCollision] = setupCollisionWritten,
[PakAssetType.CellStructureCollision] =
cellStructureWritten,
[PakAssetType.EnvCellTopology] = envCellTopologyWritten,
};
var report = new BakeReport
{
Header = header,
@ -382,9 +794,27 @@ public static class BakeRunner
EnvCellKeys = envCellKeysWritten,
UniqueEnvCellGeometries = envCellGeometriesWritten,
EnvCellAliases = envCellKeysWritten - envCellGeometriesWritten,
GfxObjCollisionKeys = gfxCollisionWritten,
UniqueGfxObjCollisions = uniqueGfxCollisions,
GfxObjCollisionAliases = gfxCollisionAliasCount,
SetupCollisionKeys = setupCollisionWritten,
UniqueSetupCollisions = uniqueSetupCollisions,
SetupCollisionAliases = setupCollisionAliasCount,
CellStructureCollisionKeys = cellStructureWritten,
UniqueCellStructureCollisions = uniqueCellStructures,
CellStructureCollisionAliases =
cellStructureAliasCount,
EnvCellTopologyKeys = envCellTopologyWritten,
SideStagedKeys = sideStagedWritten,
PhysicalBlobs =
gfxObjWritten + setupWritten + envCellGeometriesWritten + sideStagedWritten,
gfxObjWritten
+ setupWritten
+ envCellGeometriesWritten
+ sideStagedWritten
+ uniqueGfxCollisions
+ uniqueSetupCollisions
+ uniqueCellStructures
+ envCellTopologyWritten,
TotalKeys = writtenKeys.Count,
Failures = failures.Count,
ExtractionAndWriteElapsed = stopwatch.Elapsed,
@ -392,6 +822,7 @@ public static class BakeRunner
OutputBytes = new FileInfo(options.OutPath).Length,
PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64,
TypeCounts = typeCounts,
};
report = report with { SideStagedDuplicateKeys = sideStagedDuped };
@ -400,6 +831,63 @@ public static class BakeRunner
}
}
private static bool WriteExactPayload(
PakWriter writer,
ulong key,
byte[] payload,
ExactPayloadAliasCatalog aliases)
{
if (aliases.TryFind(payload, out ulong primaryKey))
{
writer.AddAlias(key, primaryKey);
return true;
}
writer.AddBlob(key, payload);
aliases.Add(key, payload);
return false;
}
private static (int Physical, int Aliases) WriteExactPayloadGroup(
PakWriter writer,
PakAssetType type,
IReadOnlyList<uint> fileIds,
byte[] payload,
ExactPayloadAliasCatalog aliases,
HashSet<ulong> writtenKeys)
{
if (fileIds.Count == 0)
throw new InvalidDataException("collision alias group is empty");
int firstUnwrittenIndex = 0;
ulong primaryKey;
int physical;
if (aliases.TryFind(payload, out primaryKey))
{
physical = 0;
}
else
{
primaryKey = PakKey.Compose(type, fileIds[0]);
writer.AddBlob(primaryKey, payload);
aliases.Add(primaryKey, payload);
writtenKeys.Add(primaryKey);
firstUnwrittenIndex = 1;
physical = 1;
}
int aliasCount = 0;
for (int i = firstUnwrittenIndex; i < fileIds.Count; i++)
{
ulong aliasKey = PakKey.Compose(type, fileIds[i]);
writer.AddAlias(aliasKey, primaryKey);
writtenKeys.Add(aliasKey);
aliasCount++;
}
return (physical, aliasCount);
}
private static int DrainSideStaged(
ConcurrentQueue<ObjectMeshData> sideStaged,
Dictionary<ulong, ObjectMeshData> sideStagedByKey,
@ -450,6 +938,20 @@ public static class BakeRunner
Console.WriteLine($" EnvCell unique: {report.UniqueEnvCellGeometries:N0}");
Console.WriteLine($" EnvCell aliases: {report.EnvCellAliases:N0}");
Console.WriteLine($" EnvCell dedup: {report.EnvCellDedupRatio:F1}x");
Console.WriteLine(
$" Gfx collisions: {report.GfxObjCollisionKeys:N0} keys, " +
$"{report.UniqueGfxObjCollisions:N0} blobs, " +
$"{report.GfxObjCollisionAliases:N0} aliases");
Console.WriteLine(
$" Setup collisions: {report.SetupCollisionKeys:N0} keys, " +
$"{report.UniqueSetupCollisions:N0} blobs, " +
$"{report.SetupCollisionAliases:N0} aliases");
Console.WriteLine(
$" Cell structures: {report.CellStructureCollisionKeys:N0} keys, " +
$"{report.UniqueCellStructureCollisions:N0} blobs, " +
$"{report.CellStructureCollisionAliases:N0} aliases");
Console.WriteLine(
$" Cell topologies: {report.EnvCellTopologyKeys:N0}");
Console.WriteLine(
$" side-staged keys: {report.SideStagedKeys:N0} " +
$"({report.SideStagedDuplicateKeys:N0} duplicate keys)");
@ -526,4 +1028,15 @@ public static class BakeRunner
return envCellIds;
}
private sealed record CollisionPayloadResult(
ulong Key,
PakAssetType Type,
uint FileId,
byte[] Payload);
private sealed record CellStructurePayloadResult(
EnvCellGeometryGroup Group,
FlatCellStructureCollisionAsset Structure,
byte[] Payload);
}

50
src/AcDream.Bake/ExactPayloadAliasCatalog.cs Normal file
View file

@ -0,0 +1,50 @@
using System.Security.Cryptography;
namespace AcDream.Bake;
/// <summary>
/// Offline-only exact-byte deduplication catalog. SHA-256 narrows candidates;
/// a complete byte comparison is still mandatory before any alias is emitted.
/// </summary>
internal sealed class ExactPayloadAliasCatalog
{
private readonly Dictionary<string, List<Entry>> _entriesByDigest =
new(StringComparer.Ordinal);
public bool TryFind(ReadOnlySpan<byte> payload, out ulong primaryKey)
{
string digest = Digest(payload);
if (_entriesByDigest.TryGetValue(digest, out List<Entry>? candidates))
{
foreach (Entry candidate in candidates)
{
if (payload.SequenceEqual(candidate.Payload))
{
primaryKey = candidate.PrimaryKey;
return true;
}
}
}
primaryKey = 0;
return false;
}
public void Add(ulong primaryKey, byte[] payload)
{
ArgumentNullException.ThrowIfNull(payload);
string digest = Digest(payload);
if (!_entriesByDigest.TryGetValue(digest, out List<Entry>? entries))
{
entries = [];
_entriesByDigest.Add(digest, entries);
}
entries.Add(new Entry(primaryKey, payload));
}
private static string Digest(ReadOnlySpan<byte> payload) =>
Convert.ToHexString(SHA256.HashData(payload));
private sealed record Entry(ulong PrimaryKey, byte[] Payload);
}