Optimize prepared asset package v2
This commit is contained in:
parent
d123c4b67c
commit
0dd966f3a0
28 changed files with 1277 additions and 106 deletions
|
|
@ -1,6 +1,7 @@
|
|||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using System.Numerics;
|
||||
using System.Runtime;
|
||||
using AcDream.Content;
|
||||
using AcDream.Content.Pak;
|
||||
using AcDream.Core.Physics;
|
||||
|
|
@ -44,6 +45,7 @@ public sealed record BakeReport
|
|||
public required int UniqueCellStructureCollisions { get; init; }
|
||||
public required int CellStructureCollisionAliases { get; init; }
|
||||
public required int EnvCellTopologyKeys { get; init; }
|
||||
public required int TexturePayloadKeys { get; init; }
|
||||
public required int SideStagedKeys { get; init; }
|
||||
public int SideStagedDuplicateKeys { get; init; }
|
||||
public required int PhysicalBlobs { get; init; }
|
||||
|
|
@ -54,6 +56,9 @@ public sealed record BakeReport
|
|||
public required long OutputBytes { get; init; }
|
||||
public required long PeakWorkingSetBytes { get; init; }
|
||||
public required long PeakPrivateBytes { get; init; }
|
||||
public required long DecodedPayloadBytes { get; init; }
|
||||
public required long StoredPayloadBytes { get; init; }
|
||||
public required int CompressedBlobs { get; init; }
|
||||
public required IReadOnlyDictionary<PakAssetType, int> TypeCounts
|
||||
{
|
||||
get;
|
||||
|
|
@ -62,6 +67,9 @@ public sealed record BakeReport
|
|||
|
||||
public double EnvCellDedupRatio =>
|
||||
UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries;
|
||||
|
||||
public double PayloadCompressionRatio =>
|
||||
StoredPayloadBytes == 0 ? 0 : (double)DecodedPayloadBytes / StoredPayloadBytes;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -76,6 +84,7 @@ public static class BakeRunner
|
|||
/// Bounds peak decoded output while keeping workers busy.
|
||||
/// </summary>
|
||||
private const int BatchSize = 16;
|
||||
private const int TransientCollectionStride = 2_048;
|
||||
|
||||
public static int Run(BakeOptions options)
|
||||
{
|
||||
|
|
@ -183,6 +192,9 @@ public static class BakeRunner
|
|||
ordinaryWork.AddRange(
|
||||
setupIds.Select(
|
||||
id => (PakKey.Compose(PakAssetType.SetupMesh, id), PakAssetType.SetupMesh, id)));
|
||||
var scheduledMeshKeys = ordinaryWork
|
||||
.Select(item => item.Key)
|
||||
.ToHashSet();
|
||||
ordinaryWork.Sort((a, b) => a.Key.CompareTo(b.Key));
|
||||
|
||||
Console.WriteLine(
|
||||
|
|
@ -199,6 +211,7 @@ public static class BakeRunner
|
|||
|
||||
var header = new PakHeader
|
||||
{
|
||||
FormatVersion = PakFormat.CurrentFormatVersion,
|
||||
PortalIteration = (uint)dats.Portal.Iteration!.CurrentIteration,
|
||||
CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration,
|
||||
HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration,
|
||||
|
|
@ -272,7 +285,11 @@ public static class BakeRunner
|
|||
sideStagedDuped += DrainSideStaged(
|
||||
sideStaged,
|
||||
sideStagedByKey,
|
||||
writtenKeys);
|
||||
writtenKeys,
|
||||
scheduledMeshKeys);
|
||||
CompactTransientBakeBuffersIfDue(
|
||||
batchStart + batch.Length,
|
||||
final: batchStart + BatchSize >= ordinaryWork.Count);
|
||||
ReportProgressIfDue(
|
||||
completed,
|
||||
totalExtractionJobs,
|
||||
|
|
@ -373,7 +390,11 @@ public static class BakeRunner
|
|||
sideStagedDuped += DrainSideStaged(
|
||||
sideStaged,
|
||||
sideStagedByKey,
|
||||
writtenKeys);
|
||||
writtenKeys,
|
||||
scheduledMeshKeys);
|
||||
CompactTransientBakeBuffersIfDue(
|
||||
batchStart + batch.Length,
|
||||
final: batchStart + BatchSize >= envCatalog.Groups.Count);
|
||||
ReportProgressIfDue(
|
||||
completed,
|
||||
totalExtractionJobs,
|
||||
|
|
@ -801,6 +822,7 @@ public static class BakeRunner
|
|||
[PakAssetType.CellStructureCollision] =
|
||||
cellStructureWritten,
|
||||
[PakAssetType.EnvCellTopology] = envCellTopologyWritten,
|
||||
[PakAssetType.TexturePayload] = writer.TextureBlobCount,
|
||||
};
|
||||
var report = new BakeReport
|
||||
{
|
||||
|
|
@ -821,23 +843,19 @@ public static class BakeRunner
|
|||
CellStructureCollisionAliases =
|
||||
cellStructureAliasCount,
|
||||
EnvCellTopologyKeys = envCellTopologyWritten,
|
||||
TexturePayloadKeys = writer.TextureBlobCount,
|
||||
SideStagedKeys = sideStagedWritten,
|
||||
PhysicalBlobs =
|
||||
gfxObjWritten
|
||||
+ setupWritten
|
||||
+ envCellGeometriesWritten
|
||||
+ sideStagedWritten
|
||||
+ uniqueGfxCollisions
|
||||
+ uniqueSetupCollisions
|
||||
+ uniqueCellStructures
|
||||
+ envCellTopologyWritten,
|
||||
TotalKeys = writtenKeys.Count,
|
||||
PhysicalBlobs = writer.PhysicalBlobCount,
|
||||
TotalKeys = writer.EntryCount,
|
||||
Failures = failures.Count,
|
||||
ExtractionAndWriteElapsed = stopwatch.Elapsed,
|
||||
Elapsed = stopwatch.Elapsed,
|
||||
OutputBytes = new FileInfo(options.OutPath).Length,
|
||||
PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
|
||||
PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64,
|
||||
DecodedPayloadBytes = writer.DecodedPayloadBytes,
|
||||
StoredPayloadBytes = writer.StoredPayloadBytes,
|
||||
CompressedBlobs = writer.CompressedBlobCount,
|
||||
TypeCounts = typeCounts,
|
||||
};
|
||||
|
||||
|
|
@ -907,19 +925,60 @@ public static class BakeRunner
|
|||
private static int DrainSideStaged(
|
||||
ConcurrentQueue<ObjectMeshData> sideStaged,
|
||||
Dictionary<ulong, ObjectMeshData> sideStagedByKey,
|
||||
HashSet<ulong> writtenKeys)
|
||||
HashSet<ulong> writtenKeys,
|
||||
HashSet<ulong> scheduledMeshKeys)
|
||||
{
|
||||
int duplicates = 0;
|
||||
while (sideStaged.TryDequeue(out var staged))
|
||||
{
|
||||
uint fileId = (uint)(staged.ObjectId & 0xFFFF_FFFFu);
|
||||
ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId);
|
||||
if (writtenKeys.Contains(key) || !sideStagedByKey.TryAdd(key, staged))
|
||||
// A full bake already schedules every GfxObj. Retaining an early
|
||||
// particle-preload copy until that later row was written pinned
|
||||
// thousands of complete meshes and several GiB of texture arrays.
|
||||
// Only filtered bakes need to retain a side-staged key that is not
|
||||
// otherwise in their work list.
|
||||
if (writtenKeys.Contains(key)
|
||||
|| scheduledMeshKeys.Contains(key)
|
||||
|| !sideStagedByKey.TryAdd(key, staged))
|
||||
duplicates++;
|
||||
}
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Mesh decode and compression use large temporary pixel/serialization
|
||||
/// arrays. A short offline bake otherwise finishes before workstation GC
|
||||
/// decides to compact the LOH, leaving several GiB committed despite the
|
||||
/// live set being bounded. Batch boundaries have no active workers and
|
||||
/// are the deterministic safe point to reclaim those transients. The
|
||||
/// stride keeps the CPU cost small while bounding peak memory.
|
||||
/// </summary>
|
||||
private static void CompactTransientBakeBuffersIfDue(
|
||||
int completedItems,
|
||||
bool final)
|
||||
{
|
||||
if (!final && completedItems % TransientCollectionStride != 0)
|
||||
return;
|
||||
|
||||
GCLargeObjectHeapCompactionMode previous =
|
||||
GCSettings.LargeObjectHeapCompactionMode;
|
||||
try
|
||||
{
|
||||
GCSettings.LargeObjectHeapCompactionMode =
|
||||
GCLargeObjectHeapCompactionMode.CompactOnce;
|
||||
GC.Collect(
|
||||
GC.MaxGeneration,
|
||||
GCCollectionMode.Forced,
|
||||
blocking: true,
|
||||
compacting: true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
GCSettings.LargeObjectHeapCompactionMode = previous;
|
||||
}
|
||||
}
|
||||
|
||||
private static void ReportProgressIfDue(
|
||||
long done,
|
||||
int total,
|
||||
|
|
@ -977,11 +1036,18 @@ public static class BakeRunner
|
|||
$"{report.CellStructureCollisionAliases:N0} aliases");
|
||||
Console.WriteLine(
|
||||
$" Cell topologies: {report.EnvCellTopologyKeys:N0}");
|
||||
Console.WriteLine(
|
||||
$" texture payloads: {report.TexturePayloadKeys:N0}");
|
||||
Console.WriteLine(
|
||||
$" side-staged keys: {report.SideStagedKeys:N0} " +
|
||||
$"({report.SideStagedDuplicateKeys:N0} duplicate keys)");
|
||||
Console.WriteLine($" physical blobs: {report.PhysicalBlobs:N0}");
|
||||
Console.WriteLine($" total keys: {report.TotalKeys:N0}");
|
||||
Console.WriteLine(
|
||||
$" compressed blobs: {report.CompressedBlobs:N0}; " +
|
||||
$"payload {report.DecodedPayloadBytes / 1024.0 / 1024.0:F1} -> " +
|
||||
$"{report.StoredPayloadBytes / 1024.0 / 1024.0:F1} MB " +
|
||||
$"({report.PayloadCompressionRatio:F2}x)");
|
||||
Console.WriteLine($" failures: {report.Failures:N0}");
|
||||
Console.WriteLine(
|
||||
$" extract + write: {report.ExtractionAndWriteElapsed.TotalSeconds:F1} s");
|
||||
|
|
|
|||
|
|
@ -433,7 +433,18 @@ public sealed class MeshExtractor {
|
|||
isClipMap,
|
||||
surface.Type.HasFlag(SurfaceType.Additive));
|
||||
|
||||
if (TextureHelpers.IsCompressedFormat(renderSurface.Format)) {
|
||||
if (CanPreserveCompressedTexture(
|
||||
renderSurface.Format,
|
||||
isClipMap,
|
||||
surface.Translucency)) {
|
||||
// Vulkan requires textureCompressionBC, so an unedited
|
||||
// DAT DXT surface can stay byte-exact all the way to the
|
||||
// GPU. Decoding it here inflated pak, CPU, and GPU
|
||||
// residency by up to eight times.
|
||||
textureFormat = ToTextureFormat(renderSurface.Format);
|
||||
textureData = renderSurface.SourceData;
|
||||
}
|
||||
else if (TextureHelpers.IsCompressedFormat(renderSurface.Format)) {
|
||||
isDxt3or5 = renderSurface.Format == DatReaderWriter.Enums.PixelFormat.PFID_DXT3 || renderSurface.Format == DatReaderWriter.Enums.PixelFormat.PFID_DXT5;
|
||||
textureFormat = TextureFormat.RGBA8;
|
||||
uploadPixelFormat = UploadPixelFormat.Rgba;
|
||||
|
|
@ -822,7 +833,14 @@ public sealed class MeshExtractor {
|
|||
isClipMap,
|
||||
surface.Type.HasFlag(SurfaceType.Additive));
|
||||
|
||||
if (_decodedTextureCache.TryGet(decodedTextureKey, out var cachedData)) {
|
||||
if (CanPreserveCompressedTexture(
|
||||
renderSurface.Format,
|
||||
isClipMap,
|
||||
surface.Translucency)) {
|
||||
textureData = renderSurface.SourceData;
|
||||
textureFormat = ToTextureFormat(renderSurface.Format);
|
||||
}
|
||||
else if (_decodedTextureCache.TryGet(decodedTextureKey, out var cachedData)) {
|
||||
textureData = cachedData;
|
||||
textureFormat = TextureFormat.RGBA8;
|
||||
uploadPixelFormat = UploadPixelFormat.Rgba;
|
||||
|
|
@ -1126,6 +1144,26 @@ public sealed class MeshExtractor {
|
|||
return textureData;
|
||||
}
|
||||
|
||||
private static bool CanPreserveCompressedTexture(
|
||||
DatReaderWriter.Enums.PixelFormat format,
|
||||
bool isClipMap,
|
||||
float translucency) =>
|
||||
TextureHelpers.IsCompressedFormat(format)
|
||||
&& !isClipMap
|
||||
&& translucency <= 0.0f;
|
||||
|
||||
private static TextureFormat ToTextureFormat(
|
||||
DatReaderWriter.Enums.PixelFormat format) => format switch
|
||||
{
|
||||
DatReaderWriter.Enums.PixelFormat.PFID_DXT1 => TextureFormat.DXT1,
|
||||
DatReaderWriter.Enums.PixelFormat.PFID_DXT3 => TextureFormat.DXT3,
|
||||
DatReaderWriter.Enums.PixelFormat.PFID_DXT5 => TextureFormat.DXT5,
|
||||
_ => throw new ArgumentOutOfRangeException(
|
||||
nameof(format),
|
||||
format,
|
||||
"The source is not a supported block-compressed texture."),
|
||||
};
|
||||
|
||||
private void BuildPolygonIndices(Polygon poly, GfxObj gfxObj, Vector3 scale,
|
||||
Dictionary<(ushort vertId, ushort uvIdx, bool isNeg), ushort> UVLookup,
|
||||
List<VertexPositionNormalTexture> vertices, List<ushort> indices, bool useNegSurface, ref bool hasWrappingUVs) {
|
||||
|
|
|
|||
|
|
@ -36,34 +36,63 @@ namespace AcDream.Content.Pak;
|
|||
public static class ObjectMeshDataSerializer {
|
||||
public static void Write(ObjectMeshData data, Stream stream) {
|
||||
using var bw = new BinaryWriter(stream, System.Text.Encoding.UTF8, leaveOpen: true);
|
||||
WriteObjectMeshData(bw, data);
|
||||
WriteObjectMeshData(bw, data, externalTextures: null);
|
||||
}
|
||||
|
||||
public static void WriteExternalTextures(
|
||||
ObjectMeshData data,
|
||||
Stream stream,
|
||||
Func<TextureKey, byte[], ulong> registerTexture) {
|
||||
ArgumentNullException.ThrowIfNull(registerTexture);
|
||||
using var bw = new BinaryWriter(
|
||||
stream,
|
||||
System.Text.Encoding.UTF8,
|
||||
leaveOpen: true);
|
||||
WriteObjectMeshData(bw, data, registerTexture);
|
||||
}
|
||||
|
||||
/// <summary>Deserializes directly over <paramref name="bytes"/> — no defensive copy (PakReader's single-pass read reuses its CRC buffer here).</summary>
|
||||
public static ObjectMeshData Read(byte[] bytes) {
|
||||
using var ms = new MemoryStream(bytes, writable: false);
|
||||
using var br = new BinaryReader(ms, System.Text.Encoding.UTF8, leaveOpen: true);
|
||||
return ReadObjectMeshData(br);
|
||||
return ReadObjectMeshData(br, externalTextures: null);
|
||||
}
|
||||
|
||||
public static ObjectMeshData Read(ReadOnlySpan<byte> bytes) => Read(bytes.ToArray());
|
||||
|
||||
public static ObjectMeshData ReadExternalTextures(
|
||||
byte[] bytes,
|
||||
Func<ulong, byte[]> resolveTexture) {
|
||||
ArgumentNullException.ThrowIfNull(resolveTexture);
|
||||
using var ms = new MemoryStream(bytes, writable: false);
|
||||
using var br = new BinaryReader(
|
||||
ms,
|
||||
System.Text.Encoding.UTF8,
|
||||
leaveOpen: true);
|
||||
return ReadObjectMeshData(br, resolveTexture);
|
||||
}
|
||||
|
||||
// ---- ObjectMeshData -----------------------------------------------------
|
||||
|
||||
private static void WriteObjectMeshData(BinaryWriter w, ObjectMeshData data) {
|
||||
private static void WriteObjectMeshData(
|
||||
BinaryWriter w,
|
||||
ObjectMeshData data,
|
||||
Func<TextureKey, byte[], ulong>? externalTextures) {
|
||||
w.Write(data.ObjectId);
|
||||
w.Write(data.IsSetup);
|
||||
|
||||
WriteVertexArray(w, data.Vertices);
|
||||
|
||||
w.Write(data.Batches.Count);
|
||||
foreach (var batch in data.Batches) WriteMeshBatchData(w, batch);
|
||||
foreach (var batch in data.Batches)
|
||||
WriteMeshBatchData(w, batch, externalTextures);
|
||||
|
||||
w.Write(data.UploadAttempts);
|
||||
|
||||
// EnvCellGeometry: recursive nested block.
|
||||
w.Write(data.EnvCellGeometry is not null);
|
||||
if (data.EnvCellGeometry is not null) WriteObjectMeshData(w, data.EnvCellGeometry);
|
||||
if (data.EnvCellGeometry is not null)
|
||||
WriteObjectMeshData(w, data.EnvCellGeometry, externalTextures);
|
||||
|
||||
w.Write(data.SetupParts.Count);
|
||||
foreach (var (gfxObjId, transform) in data.SetupParts) {
|
||||
|
|
@ -74,7 +103,7 @@ public static class ObjectMeshDataSerializer {
|
|||
w.Write(data.ParticleEmitters.Count);
|
||||
foreach (var emitter in data.ParticleEmitters) WriteStagedEmitter(w, emitter);
|
||||
|
||||
WriteTextureBatches(w, data.TextureBatches);
|
||||
WriteTextureBatches(w, data.TextureBatches, externalTextures);
|
||||
|
||||
WriteBoundingBox(w, data.BoundingBox);
|
||||
WriteVector3(w, data.SortCenter);
|
||||
|
|
@ -86,7 +115,9 @@ public static class ObjectMeshDataSerializer {
|
|||
WriteVector3Array(w, data.EdgeLines);
|
||||
}
|
||||
|
||||
private static ObjectMeshData ReadObjectMeshData(BinaryReader r) {
|
||||
private static ObjectMeshData ReadObjectMeshData(
|
||||
BinaryReader r,
|
||||
Func<ulong, byte[]>? externalTextures) {
|
||||
var data = new ObjectMeshData {
|
||||
ObjectId = r.ReadUInt64(),
|
||||
IsSetup = r.ReadBoolean(),
|
||||
|
|
@ -96,13 +127,16 @@ public static class ObjectMeshDataSerializer {
|
|||
|
||||
int batchCount = r.ReadInt32();
|
||||
var batches = new List<MeshBatchData>(batchCount);
|
||||
for (int i = 0; i < batchCount; i++) batches.Add(ReadMeshBatchData(r));
|
||||
for (int i = 0; i < batchCount; i++)
|
||||
batches.Add(ReadMeshBatchData(r, externalTextures));
|
||||
data.Batches = batches;
|
||||
|
||||
data.UploadAttempts = r.ReadInt32();
|
||||
|
||||
bool hasEnvCellGeometry = r.ReadBoolean();
|
||||
data.EnvCellGeometry = hasEnvCellGeometry ? ReadObjectMeshData(r) : null;
|
||||
data.EnvCellGeometry = hasEnvCellGeometry
|
||||
? ReadObjectMeshData(r, externalTextures)
|
||||
: null;
|
||||
|
||||
int setupPartCount = r.ReadInt32();
|
||||
var setupParts = new List<(ulong GfxObjId, Matrix4x4 Transform)>(setupPartCount);
|
||||
|
|
@ -118,7 +152,7 @@ public static class ObjectMeshDataSerializer {
|
|||
for (int i = 0; i < emitterCount; i++) emitters.Add(ReadStagedEmitter(r));
|
||||
data.ParticleEmitters = emitters;
|
||||
|
||||
data.TextureBatches = ReadTextureBatches(r);
|
||||
data.TextureBatches = ReadTextureBatches(r, externalTextures);
|
||||
|
||||
data.BoundingBox = ReadBoundingBox(r);
|
||||
data.SortCenter = ReadVector3(r);
|
||||
|
|
@ -134,24 +168,37 @@ public static class ObjectMeshDataSerializer {
|
|||
|
||||
// ---- MeshBatchData -------------------------------------------------------
|
||||
|
||||
private static void WriteMeshBatchData(BinaryWriter w, MeshBatchData batch) {
|
||||
private static void WriteMeshBatchData(
|
||||
BinaryWriter w,
|
||||
MeshBatchData batch,
|
||||
Func<TextureKey, byte[], ulong>? externalTextures) {
|
||||
WriteUInt16Array(w, batch.Indices);
|
||||
WriteTextureFormatTuple(w, batch.TextureFormat);
|
||||
WriteTextureKey(w, batch.TextureKey);
|
||||
w.Write(batch.TextureIndex);
|
||||
WriteByteArray(w, batch.TextureData);
|
||||
WriteTexturePayload(
|
||||
w,
|
||||
batch.TextureKey,
|
||||
batch.TextureData,
|
||||
externalTextures);
|
||||
WriteNullableInt32Enum(w, batch.UploadPixelFormat.HasValue, batch.UploadPixelFormat is { } upf ? (int)upf : 0);
|
||||
WriteNullableInt32Enum(w, batch.UploadPixelType.HasValue, batch.UploadPixelType is { } upt ? (int)upt : 0);
|
||||
w.Write((int)batch.CullMode);
|
||||
}
|
||||
|
||||
private static MeshBatchData ReadMeshBatchData(BinaryReader r) {
|
||||
private static MeshBatchData ReadMeshBatchData(
|
||||
BinaryReader r,
|
||||
Func<ulong, byte[]>? externalTextures) {
|
||||
ushort[] indices = ReadUInt16Array(r);
|
||||
(int Width, int Height, TextureFormat Format) format =
|
||||
ReadTextureFormatTuple(r);
|
||||
TextureKey key = ReadTextureKey(r);
|
||||
var batch = new MeshBatchData {
|
||||
Indices = ReadUInt16Array(r),
|
||||
TextureFormat = ReadTextureFormatTuple(r),
|
||||
TextureKey = ReadTextureKey(r),
|
||||
Indices = indices,
|
||||
TextureFormat = format,
|
||||
TextureKey = key,
|
||||
TextureIndex = r.ReadInt32(),
|
||||
TextureData = ReadByteArray(r),
|
||||
TextureData = ReadTexturePayload(r, externalTextures),
|
||||
};
|
||||
batch.UploadPixelFormat = ReadNullableInt32Enum(r, v => (UploadPixelFormat)v);
|
||||
batch.UploadPixelType = ReadNullableInt32Enum(r, v => (UploadPixelType)v);
|
||||
|
|
@ -161,9 +208,12 @@ public static class ObjectMeshDataSerializer {
|
|||
|
||||
// ---- TextureBatchData / TextureBatches dictionary -------------------------
|
||||
|
||||
private static void WriteTextureBatchData(BinaryWriter w, TextureBatchData batch) {
|
||||
private static void WriteTextureBatchData(
|
||||
BinaryWriter w,
|
||||
TextureBatchData batch,
|
||||
Func<TextureKey, byte[], ulong>? externalTextures) {
|
||||
WriteTextureKey(w, batch.Key);
|
||||
WriteByteArray(w, batch.TextureData);
|
||||
WriteTexturePayload(w, batch.Key, batch.TextureData, externalTextures);
|
||||
WriteNullableInt32Enum(w, batch.UploadPixelFormat.HasValue, batch.UploadPixelFormat is { } upf ? (int)upf : 0);
|
||||
WriteNullableInt32Enum(w, batch.UploadPixelType.HasValue, batch.UploadPixelType is { } upt ? (int)upt : 0);
|
||||
WriteUInt16List(w, batch.Indices);
|
||||
|
|
@ -174,10 +224,13 @@ public static class ObjectMeshDataSerializer {
|
|||
w.Write(batch.HasWrappingUVs);
|
||||
}
|
||||
|
||||
private static TextureBatchData ReadTextureBatchData(BinaryReader r) {
|
||||
private static TextureBatchData ReadTextureBatchData(
|
||||
BinaryReader r,
|
||||
Func<ulong, byte[]>? externalTextures) {
|
||||
TextureKey key = ReadTextureKey(r);
|
||||
var batch = new TextureBatchData {
|
||||
Key = ReadTextureKey(r),
|
||||
TextureData = ReadByteArray(r),
|
||||
Key = key,
|
||||
TextureData = ReadTexturePayload(r, externalTextures),
|
||||
};
|
||||
batch.UploadPixelFormat = ReadNullableInt32Enum(r, v => (UploadPixelFormat)v);
|
||||
batch.UploadPixelType = ReadNullableInt32Enum(r, v => (UploadPixelType)v);
|
||||
|
|
@ -198,7 +251,8 @@ public static class ObjectMeshDataSerializer {
|
|||
/// </summary>
|
||||
private static void WriteTextureBatches(
|
||||
BinaryWriter w,
|
||||
Dictionary<(int Width, int Height, TextureFormat Format), List<TextureBatchData>> batches) {
|
||||
Dictionary<(int Width, int Height, TextureFormat Format), List<TextureBatchData>> batches,
|
||||
Func<TextureKey, byte[], ulong>? externalTextures) {
|
||||
var sortedKeys = batches.Keys
|
||||
.OrderBy(k => k.Width)
|
||||
.ThenBy(k => k.Height)
|
||||
|
|
@ -213,11 +267,14 @@ public static class ObjectMeshDataSerializer {
|
|||
|
||||
var list = batches[key];
|
||||
w.Write(list.Count);
|
||||
foreach (var item in list) WriteTextureBatchData(w, item);
|
||||
foreach (var item in list)
|
||||
WriteTextureBatchData(w, item, externalTextures);
|
||||
}
|
||||
}
|
||||
|
||||
private static Dictionary<(int Width, int Height, TextureFormat Format), List<TextureBatchData>> ReadTextureBatches(BinaryReader r) {
|
||||
private static Dictionary<(int Width, int Height, TextureFormat Format), List<TextureBatchData>> ReadTextureBatches(
|
||||
BinaryReader r,
|
||||
Func<ulong, byte[]>? externalTextures) {
|
||||
int groupCount = r.ReadInt32();
|
||||
var result = new Dictionary<(int Width, int Height, TextureFormat Format), List<TextureBatchData>>(groupCount);
|
||||
for (int i = 0; i < groupCount; i++) {
|
||||
|
|
@ -227,13 +284,32 @@ public static class ObjectMeshDataSerializer {
|
|||
|
||||
int listCount = r.ReadInt32();
|
||||
var list = new List<TextureBatchData>(listCount);
|
||||
for (int j = 0; j < listCount; j++) list.Add(ReadTextureBatchData(r));
|
||||
for (int j = 0; j < listCount; j++)
|
||||
list.Add(ReadTextureBatchData(r, externalTextures));
|
||||
|
||||
result[(width, height, format)] = list;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void WriteTexturePayload(
|
||||
BinaryWriter writer,
|
||||
TextureKey key,
|
||||
byte[] bytes,
|
||||
Func<TextureKey, byte[], ulong>? externalTextures) {
|
||||
if (externalTextures is null)
|
||||
WriteByteArray(writer, bytes);
|
||||
else
|
||||
writer.Write(externalTextures(key, bytes));
|
||||
}
|
||||
|
||||
private static byte[] ReadTexturePayload(
|
||||
BinaryReader reader,
|
||||
Func<ulong, byte[]>? externalTextures) =>
|
||||
externalTextures is null
|
||||
? ReadByteArray(reader)
|
||||
: externalTextures(reader.ReadUInt64());
|
||||
|
||||
// ---- StagedEmitter / ParticleEmitter (DBObj) ------------------------------
|
||||
|
||||
private static void WriteStagedEmitter(BinaryWriter w, StagedEmitter emitter) {
|
||||
|
|
|
|||
124
src/AcDream.Content/Pak/PakBlobCodec.cs
Normal file
124
src/AcDream.Content/Pak/PakBlobCodec.cs
Normal file
|
|
@ -0,0 +1,124 @@
|
|||
using System.Buffers;
|
||||
using System.Buffers.Binary;
|
||||
using System.IO.Compression;
|
||||
|
||||
namespace AcDream.Content.Pak;
|
||||
|
||||
/// <summary>
|
||||
/// Format-2 independent-blob compression. Each payload remains separately
|
||||
/// addressable; incompressible and small payloads stay raw so a size win can
|
||||
/// never turn into needless runtime CPU work.
|
||||
/// </summary>
|
||||
internal static class PakBlobCodec
|
||||
{
|
||||
// The largest format-1 payload in the complete installed package is
|
||||
// 19,683,674 bytes. Texture externalization makes format-2 mesh blobs
|
||||
// smaller still. A fixed 64 MiB ceiling prevents a corrupt package from
|
||||
// declaring a multi-gigabyte allocation before Brotli can reject it.
|
||||
internal const int MaximumDecodedBytes = 64 * 1024 * 1024;
|
||||
private const int DecodedLengthPrefixSize = sizeof(uint);
|
||||
private const int MinimumCompressionBytes = 512;
|
||||
private const int MinimumSavingsBytes = 64;
|
||||
private const int BrotliQuality = 1;
|
||||
private const int BrotliWindow = 22;
|
||||
|
||||
internal readonly record struct Encoded(byte[] Bytes, bool Compressed);
|
||||
|
||||
/// <summary>
|
||||
/// Attempts a worthwhile compression into an ArrayPool-owned buffer.
|
||||
/// The caller must return a non-null buffer. A false result owns no
|
||||
/// buffer and should write the original bytes directly.
|
||||
/// </summary>
|
||||
public static bool TryCompress(
|
||||
ReadOnlySpan<byte> decoded,
|
||||
out byte[]? rented,
|
||||
out int storedLength)
|
||||
{
|
||||
if (decoded.Length > MaximumDecodedBytes)
|
||||
throw new InvalidDataException(
|
||||
$"pak payload is {decoded.Length} bytes; maximum is {MaximumDecodedBytes}");
|
||||
|
||||
rented = null;
|
||||
storedLength = decoded.Length;
|
||||
if (decoded.Length < MinimumCompressionBytes)
|
||||
return false;
|
||||
|
||||
int maximum = checked(
|
||||
DecodedLengthPrefixSize
|
||||
+ BrotliEncoder.GetMaxCompressedLength(decoded.Length));
|
||||
byte[] candidate = ArrayPool<byte>.Shared.Rent(maximum);
|
||||
if (!BrotliEncoder.TryCompress(
|
||||
decoded,
|
||||
candidate.AsSpan(DecodedLengthPrefixSize, maximum - DecodedLengthPrefixSize),
|
||||
out int compressedLength,
|
||||
BrotliQuality,
|
||||
BrotliWindow))
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(candidate);
|
||||
return false;
|
||||
}
|
||||
|
||||
storedLength = checked(DecodedLengthPrefixSize + compressedLength);
|
||||
int requiredSavings = Math.Max(
|
||||
MinimumSavingsBytes,
|
||||
decoded.Length / 16);
|
||||
if (decoded.Length - storedLength < requiredSavings)
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(candidate);
|
||||
storedLength = decoded.Length;
|
||||
return false;
|
||||
}
|
||||
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(
|
||||
candidate.AsSpan(0, DecodedLengthPrefixSize),
|
||||
checked((uint)decoded.Length));
|
||||
rented = candidate;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Test/convenience wrapper. Production PakWriter uses TryCompress so it
|
||||
// can write the pooled buffer directly without a compacting copy.
|
||||
public static Encoded Encode(byte[] decoded)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(decoded);
|
||||
if (!TryCompress(decoded, out byte[]? rented, out int storedLength))
|
||||
return new Encoded(decoded, false);
|
||||
|
||||
byte[] buffer = rented
|
||||
?? throw new InvalidOperationException("compressed buffer was not returned");
|
||||
try
|
||||
{
|
||||
return new Encoded(buffer.AsSpan(0, storedLength).ToArray(), true);
|
||||
}
|
||||
finally
|
||||
{
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
}
|
||||
|
||||
public static byte[] Decode(ReadOnlySpan<byte> stored)
|
||||
{
|
||||
if (stored.Length < DecodedLengthPrefixSize)
|
||||
throw new InvalidDataException(
|
||||
"compressed pak blob is missing its decoded-length prefix");
|
||||
|
||||
uint decodedLength = BinaryPrimitives.ReadUInt32LittleEndian(
|
||||
stored[..DecodedLengthPrefixSize]);
|
||||
if (decodedLength > MaximumDecodedBytes)
|
||||
throw new InvalidDataException(
|
||||
$"compressed pak blob declares unsupported decoded length {decodedLength}");
|
||||
|
||||
var decoded = new byte[checked((int)decodedLength)];
|
||||
if (!BrotliDecoder.TryDecompress(
|
||||
stored[DecodedLengthPrefixSize..],
|
||||
decoded,
|
||||
out int written)
|
||||
|| written != decoded.Length)
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"compressed pak blob did not decode to its declared {decoded.Length} bytes");
|
||||
}
|
||||
|
||||
return decoded;
|
||||
}
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ public static class PakFormat {
|
|||
/// template); PakReader refuses to open any other version. Bump ONLY
|
||||
/// with an accompanying reader migration path.
|
||||
/// </summary>
|
||||
public const uint CurrentFormatVersion = 1;
|
||||
public const uint CurrentFormatVersion = 2;
|
||||
|
||||
/// <summary>
|
||||
/// Identity of the bake algorithm that produced the payloads. Version 2
|
||||
|
|
@ -28,9 +28,11 @@ public static class PakFormat {
|
|||
/// zero vertices on its positive side, so any pak baked by an older tool
|
||||
/// is missing those faces (e.g. the Holtburg windmill axle 0x010010CE,
|
||||
/// 8 polygons all NoPos + Base1Solid, extracted to a 0-vertex mesh). The
|
||||
/// binary format remains version 1.
|
||||
/// binary format remains version 1. Version 6 introduces pak format 2:
|
||||
/// globally shared texture payloads plus independently Brotli-compressed
|
||||
/// blobs with raw fallback. Mesh geometry and decoded pixels remain exact.
|
||||
/// </summary>
|
||||
public const uint CurrentBakeToolVersion = 5;
|
||||
public const uint CurrentBakeToolVersion = 6;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -38,7 +40,7 @@ public static class PakFormat {
|
|||
/// <code>
|
||||
/// offset size field
|
||||
/// 0 4 magic 'ACPK' (0x4B504341)
|
||||
/// 4 4 formatVersion = 1
|
||||
/// 4 4 formatVersion = 2
|
||||
/// 8 4 portalIteration (DatCollection.Portal.Iteration)
|
||||
/// 12 4 cellIteration
|
||||
/// 16 4 highResIteration
|
||||
|
|
@ -48,7 +50,9 @@ public static class PakFormat {
|
|||
/// 36 4 bakeToolVersion
|
||||
/// 40 24 reserved (zero)
|
||||
/// </code>
|
||||
/// Spec: docs/superpowers/plans/2026-07-05-mp1b-pak-and-bake.md "Format v1 (normative)".
|
||||
/// Format 1 is specified by docs/superpowers/plans/2026-07-05-mp1b-pak-and-bake.md.
|
||||
/// Format 2 retains this header and adds the TOC compression flag plus global
|
||||
/// texture payload entries documented in docs/plans/2026-08-27-pak-v2-resource-campaign.md.
|
||||
/// </summary>
|
||||
public struct PakHeader {
|
||||
public const int Size = 64;
|
||||
|
|
@ -114,19 +118,32 @@ public struct PakHeader {
|
|||
}
|
||||
|
||||
/// <summary>
|
||||
/// One 24-byte TOC entry: <c>key u64, offset u64, length u32, crc32 u32</c>.
|
||||
/// One 24-byte TOC entry: <c>key u64, offset u64, lengthAndFlags u32, crc32 u32</c>.
|
||||
/// Entries in a pak's TOC are sorted ascending by <see cref="Key"/> to allow
|
||||
/// binary-search lookup. <see cref="Crc32"/> is a corruption tripwire computed
|
||||
/// over the blob bytes; the reader verifies lazily on first access.
|
||||
/// </summary>
|
||||
public struct PakTocEntry {
|
||||
public const int Size = 24;
|
||||
public const uint CompressionFlag = 0x8000_0000u;
|
||||
public const uint StoredLengthMask = 0x7FFF_FFFFu;
|
||||
|
||||
public ulong Key;
|
||||
public ulong Offset;
|
||||
public uint Length;
|
||||
public uint Crc32;
|
||||
|
||||
/// <summary>
|
||||
/// Format-2 entries use the high bit of <see cref="Length"/> to mark an
|
||||
/// independently Brotli-compressed blob. The remaining 31 bits are the
|
||||
/// exact bytes stored in the file (including the compressed blob's
|
||||
/// four-byte decoded-length prefix). Raw entries retain their historical
|
||||
/// length representation and require no second copy on read.
|
||||
/// </summary>
|
||||
public readonly bool IsCompressed => (Length & CompressionFlag) != 0;
|
||||
|
||||
public readonly uint StoredLength => Length & StoredLengthMask;
|
||||
|
||||
public void WriteTo(Span<byte> dest) {
|
||||
if (dest.Length < Size) throw new ArgumentException($"destination must be at least {Size} bytes", nameof(dest));
|
||||
BinaryPrimitives.WriteUInt64LittleEndian(dest[0..8], Key);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ public enum PakAssetType : byte {
|
|||
SetupCollision = 5,
|
||||
CellStructureCollision = 6,
|
||||
EnvCellTopology = 7,
|
||||
TexturePayload = 8,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -34,4 +35,14 @@ public static class PakKey {
|
|||
var fileId = (uint)((key >> 24) & 0xFFFFFFFFu);
|
||||
return (type, fileId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Composes a content-addressed format-2 key whose lower 56 bits are an
|
||||
/// opaque payload identity rather than the legacy file-id/reserved split.
|
||||
/// </summary>
|
||||
public static ulong ComposeOpaque(PakAssetType type, ulong payloadId) {
|
||||
if ((payloadId & 0xFF00_0000_0000_0000ul) != 0)
|
||||
throw new ArgumentOutOfRangeException(nameof(payloadId));
|
||||
return ((ulong)type << 56) | payloadId;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,6 +40,7 @@ public sealed class PakReader : IDisposable {
|
|||
private readonly MemoryMappedViewAccessor _accessor;
|
||||
private readonly long _fileLength;
|
||||
private readonly PakTocEntry[] _toc; // sorted ascending by Key
|
||||
private readonly PakTexturePayloadCache _texturePayloads = new();
|
||||
|
||||
/// <summary>Lazy per-entry verdict: absent = not yet judged, 0 = bad (bounds/crc/structure), 1 = ok.</summary>
|
||||
private readonly ConcurrentDictionary<int, int> _entryVerdictByTocIndex = new();
|
||||
|
|
@ -100,7 +101,7 @@ public sealed class PakReader : IDisposable {
|
|||
bool invalid = !IsEntryRangeValid(entry);
|
||||
if (invalid) {
|
||||
_entryVerdictByTocIndex[i] = 0;
|
||||
LogCorruptionOnce(i, $"TOC entry out of bounds (offset={entry.Offset}, length={entry.Length}, " +
|
||||
LogCorruptionOnce(i, $"TOC entry out of bounds (offset={entry.Offset}, storedLength={entry.StoredLength}, " +
|
||||
$"file={_fileLength}, toc@{Header.TocOffset})");
|
||||
}
|
||||
}
|
||||
|
|
@ -158,7 +159,9 @@ public sealed class PakReader : IDisposable {
|
|||
return status;
|
||||
|
||||
try {
|
||||
data = ObjectMeshDataSerializer.Read(bytes);
|
||||
data = ObjectMeshDataSerializer.ReadExternalTextures(
|
||||
bytes,
|
||||
ResolveTexturePayload);
|
||||
return PakObjectReadStatus.Loaded;
|
||||
}
|
||||
catch (Exception ex) {
|
||||
|
|
@ -194,8 +197,9 @@ public sealed class PakReader : IDisposable {
|
|||
// Single pass (review finding 4): ONE copy out of the map; CRC and
|
||||
// deserialization both run over this same buffer.
|
||||
ref readonly var entry = ref _toc[index];
|
||||
bytes = new byte[entry.Length];
|
||||
_accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length);
|
||||
uint storedLength = entry.StoredLength;
|
||||
bytes = new byte[checked((int)storedLength)];
|
||||
_accessor.ReadArray((long)entry.Offset, bytes, 0, checked((int)storedLength));
|
||||
|
||||
if (!judged) {
|
||||
uint actualCrc = Crc32.Compute(bytes);
|
||||
|
|
@ -208,9 +212,41 @@ public sealed class PakReader : IDisposable {
|
|||
_entryVerdictByTocIndex[index] = 1;
|
||||
}
|
||||
|
||||
if (entry.IsCompressed) {
|
||||
try {
|
||||
bytes = PakBlobCodec.Decode(bytes);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidDataException or OverflowException) {
|
||||
_entryVerdictByTocIndex[index] = 0;
|
||||
LogCorruptionOnce(
|
||||
index,
|
||||
$"decompression failed despite matching CRC: {ex.Message}");
|
||||
bytes = null;
|
||||
return PakObjectReadStatus.Corrupt;
|
||||
}
|
||||
}
|
||||
|
||||
return PakObjectReadStatus.Loaded;
|
||||
}
|
||||
|
||||
private byte[] ResolveTexturePayload(ulong key) {
|
||||
if (PakKey.Decompose(key).Type != PakAssetType.TexturePayload) {
|
||||
throw new InvalidDataException(
|
||||
$"mesh references non-texture pak key 0x{key:X16}");
|
||||
}
|
||||
|
||||
if (_texturePayloads.TryGet(key, out byte[] cachedBytes))
|
||||
return cachedBytes;
|
||||
|
||||
PakObjectReadStatus status = ReadBlobBytes(key, out byte[]? loadedBytes);
|
||||
if (status != PakObjectReadStatus.Loaded || loadedBytes is null) {
|
||||
throw new InvalidDataException(
|
||||
$"mesh references {status.ToString().ToLowerInvariant()} texture payload 0x{key:X16}");
|
||||
}
|
||||
|
||||
return _texturePayloads.AddOrGet(key, loadedBytes);
|
||||
}
|
||||
|
||||
internal void MarkPayloadCorrupt(ulong key, string reason) {
|
||||
int index = BinarySearch(key);
|
||||
if (index < 0)
|
||||
|
|
@ -269,7 +305,7 @@ public sealed class PakReader : IDisposable {
|
|||
if (!IsEntryRangeValid(entry)) {
|
||||
throw new InvalidDataException(
|
||||
$"pak TOC entry 0x{entry.Key:X16} has an invalid range " +
|
||||
$"(offset={entry.Offset}, length={entry.Length}, " +
|
||||
$"(offset={entry.Offset}, storedLength={entry.StoredLength}, " +
|
||||
$"file={_fileLength}, toc@{Header.TocOffset})");
|
||||
}
|
||||
}
|
||||
|
|
@ -288,8 +324,9 @@ public sealed class PakReader : IDisposable {
|
|||
if (_entryVerdictByTocIndex.TryGetValue(tocIndex, out var cached)) return cached;
|
||||
|
||||
ref readonly var entry = ref _toc[tocIndex];
|
||||
var bytes = new byte[entry.Length];
|
||||
_accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length);
|
||||
uint storedLength = entry.StoredLength;
|
||||
var bytes = new byte[checked((int)storedLength)];
|
||||
_accessor.ReadArray((long)entry.Offset, bytes, 0, checked((int)storedLength));
|
||||
uint actualCrc = Crc32.Compute(bytes);
|
||||
bool ok = actualCrc == entry.Crc32;
|
||||
|
||||
|
|
@ -304,7 +341,8 @@ public sealed class PakReader : IDisposable {
|
|||
if (!_loggedCorruption.TryAdd(tocIndex, true)) return;
|
||||
ref readonly var entry = ref _toc[tocIndex];
|
||||
Console.Error.WriteLine(
|
||||
$"[pak-corrupt] key 0x{entry.Key:X16} at offset {entry.Offset} (length {entry.Length}): " +
|
||||
$"[pak-corrupt] key 0x{entry.Key:X16} at offset {entry.Offset} " +
|
||||
$"(storedLength {entry.StoredLength}, compressed={entry.IsCompressed}): " +
|
||||
$"{reason} — treating as missing");
|
||||
}
|
||||
|
||||
|
|
@ -329,8 +367,11 @@ public sealed class PakReader : IDisposable {
|
|||
return false;
|
||||
}
|
||||
|
||||
return entry.Length <= Header.TocOffset - entry.Offset &&
|
||||
entry.Length <= fileLength - entry.Offset;
|
||||
if (entry.StoredLength > PakBlobCodec.MaximumDecodedBytes)
|
||||
return false;
|
||||
|
||||
return entry.StoredLength <= Header.TocOffset - entry.Offset &&
|
||||
entry.StoredLength <= fileLength - entry.Offset;
|
||||
}
|
||||
|
||||
public void Dispose() {
|
||||
|
|
|
|||
98
src/AcDream.Content/Pak/PakTexturePayloadCache.cs
Normal file
98
src/AcDream.Content/Pak/PakTexturePayloadCache.cs
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
namespace AcDream.Content.Pak;
|
||||
|
||||
/// <summary>
|
||||
/// Bounded shared-byte owner for format-2 texture payloads. Mesh objects keep
|
||||
/// ordinary array references, so evicting a cache row is safe while preventing
|
||||
/// a long travel session from pinning every texture ever observed.
|
||||
/// </summary>
|
||||
internal sealed class PakTexturePayloadCache
|
||||
{
|
||||
// Payloads are staging data on the way to GPU atlases. A 256 MiB cache
|
||||
// could retain a second world-sized texture working set after the atlas
|
||||
// already owned it; native BC makes 64 MiB ample for revisit sharing.
|
||||
internal const long DefaultMaximumBytes = 64L * 1024 * 1024;
|
||||
internal const int DefaultMaximumEntries = 1_024;
|
||||
|
||||
private readonly long _maximumBytes;
|
||||
private readonly int _maximumEntries;
|
||||
private readonly Dictionary<ulong, Entry> _entries = [];
|
||||
private readonly LinkedList<ulong> _lru = [];
|
||||
private readonly Lock _gate = new();
|
||||
private long _bytes;
|
||||
|
||||
internal int Count
|
||||
{
|
||||
get { lock (_gate) return _entries.Count; }
|
||||
}
|
||||
|
||||
internal long Bytes
|
||||
{
|
||||
get { lock (_gate) return _bytes; }
|
||||
}
|
||||
|
||||
private sealed record Entry(byte[] Bytes, LinkedListNode<ulong> Node);
|
||||
|
||||
public PakTexturePayloadCache(
|
||||
long maximumBytes = DefaultMaximumBytes,
|
||||
int maximumEntries = DefaultMaximumEntries)
|
||||
{
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(maximumBytes, 1);
|
||||
ArgumentOutOfRangeException.ThrowIfLessThan(maximumEntries, 1);
|
||||
_maximumBytes = maximumBytes;
|
||||
_maximumEntries = maximumEntries;
|
||||
}
|
||||
|
||||
public bool TryGet(ulong key, out byte[] bytes)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_entries.TryGetValue(key, out Entry? entry))
|
||||
{
|
||||
bytes = null!;
|
||||
return false;
|
||||
}
|
||||
|
||||
_lru.Remove(entry.Node);
|
||||
_lru.AddLast(entry.Node);
|
||||
bytes = entry.Bytes;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public byte[] AddOrGet(ulong key, byte[] bytes)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(bytes);
|
||||
lock (_gate)
|
||||
{
|
||||
if (_entries.TryGetValue(key, out Entry? existing))
|
||||
{
|
||||
_lru.Remove(existing.Node);
|
||||
_lru.AddLast(existing.Node);
|
||||
return existing.Bytes;
|
||||
}
|
||||
|
||||
// A single unusually large texture must not defeat the global
|
||||
// bound. Its caller still owns the returned array; it simply is
|
||||
// not retained for a later mesh.
|
||||
if (bytes.LongLength > _maximumBytes)
|
||||
return bytes;
|
||||
|
||||
LinkedListNode<ulong> node = _lru.AddLast(key);
|
||||
_entries.Add(key, new Entry(bytes, node));
|
||||
_bytes = checked(_bytes + bytes.LongLength);
|
||||
Trim();
|
||||
return bytes;
|
||||
}
|
||||
}
|
||||
|
||||
private void Trim()
|
||||
{
|
||||
while ((_bytes > _maximumBytes || _entries.Count > _maximumEntries)
|
||||
&& _lru.First is { } oldest)
|
||||
{
|
||||
_lru.RemoveFirst();
|
||||
if (_entries.Remove(oldest.Value, out Entry? removed))
|
||||
_bytes -= removed.Bytes.LongLength;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,7 +1,9 @@
|
|||
using System.Buffers;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
|
||||
namespace AcDream.Content.Pak;
|
||||
|
||||
|
|
@ -19,8 +21,16 @@ public sealed class PakWriter : IDisposable {
|
|||
private readonly List<PakTocEntry> _tocEntries = new();
|
||||
private readonly HashSet<ulong> _seenKeys = new();
|
||||
private readonly Dictionary<ulong, PakTocEntry> _receiptByKey = new();
|
||||
private readonly Dictionary<ulong, TextureIdentity> _texturePayloadByKey = new();
|
||||
private bool _finished;
|
||||
|
||||
public int TextureBlobCount { get; private set; }
|
||||
public int EntryCount => _tocEntries.Count;
|
||||
public int PhysicalBlobCount { get; private set; }
|
||||
public long DecodedPayloadBytes { get; private set; }
|
||||
public long StoredPayloadBytes { get; private set; }
|
||||
public int CompressedBlobCount { get; private set; }
|
||||
|
||||
public PakWriter(string path, PakHeader headerTemplate) {
|
||||
_stream = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None);
|
||||
_headerTemplate = headerTemplate;
|
||||
|
|
@ -49,42 +59,103 @@ public sealed class PakWriter : IDisposable {
|
|||
ReserveKey(key);
|
||||
|
||||
using var ms = new MemoryStream();
|
||||
ObjectMeshDataSerializer.Write(data, ms);
|
||||
var bytes = ms.ToArray();
|
||||
AddReservedBlob(key, bytes);
|
||||
ObjectMeshDataSerializer.WriteExternalTextures(
|
||||
data,
|
||||
ms,
|
||||
RegisterTexturePayload);
|
||||
if (!ms.TryGetBuffer(out ArraySegment<byte> buffer))
|
||||
throw new InvalidOperationException("mesh serializer did not expose its write buffer");
|
||||
AddReservedBlob(key, buffer.AsSpan(0, checked((int)ms.Length)));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds an already serialized immutable payload. This is the package seam
|
||||
/// used by typed non-render assets such as flat collision records.
|
||||
/// </summary>
|
||||
public void AddBlob(ulong key, ReadOnlySpan<byte> bytes) {
|
||||
public void AddBlob(ulong key, byte[] bytes) {
|
||||
ArgumentNullException.ThrowIfNull(bytes);
|
||||
ThrowIfFinished();
|
||||
ReserveKey(key);
|
||||
AddReservedBlob(key, bytes);
|
||||
}
|
||||
|
||||
private void AddReservedBlob(ulong key, ReadOnlySpan<byte> bytes) {
|
||||
if ((ulong)bytes.Length > uint.MaxValue) {
|
||||
throw new ArgumentException(
|
||||
"one pak payload cannot exceed UInt32.MaxValue bytes",
|
||||
nameof(bytes));
|
||||
bool compressed = PakBlobCodec.TryCompress(
|
||||
bytes,
|
||||
out byte[]? rented,
|
||||
out int storedLength);
|
||||
try
|
||||
{
|
||||
ReadOnlySpan<byte> stored = compressed
|
||||
? rented.AsSpan(0, storedLength)
|
||||
: bytes;
|
||||
long offset = _stream.Position;
|
||||
_stream.Write(stored);
|
||||
PadToAlignment();
|
||||
|
||||
var receipt = new PakTocEntry {
|
||||
Key = key,
|
||||
Offset = (ulong)offset,
|
||||
Length = checked((uint)stored.Length)
|
||||
| (compressed ? PakTocEntry.CompressionFlag : 0u),
|
||||
Crc32 = Content.Pak.Crc32.Compute(stored),
|
||||
};
|
||||
_tocEntries.Add(receipt);
|
||||
_receiptByKey.Add(key, receipt);
|
||||
PhysicalBlobCount++;
|
||||
DecodedPayloadBytes = checked(DecodedPayloadBytes + bytes.Length);
|
||||
StoredPayloadBytes = checked(StoredPayloadBytes + stored.Length);
|
||||
if (compressed)
|
||||
CompressedBlobCount++;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (rented is not null)
|
||||
ArrayPool<byte>.Shared.Return(rented);
|
||||
}
|
||||
}
|
||||
|
||||
private ulong RegisterTexturePayload(TextureKey _, byte[] bytes)
|
||||
{
|
||||
Span<byte> digest = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(bytes, digest);
|
||||
ulong payloadId =
|
||||
((ulong)digest[0] << 48)
|
||||
| ((ulong)digest[1] << 40)
|
||||
| ((ulong)digest[2] << 32)
|
||||
| ((ulong)digest[3] << 24)
|
||||
| ((ulong)digest[4] << 16)
|
||||
| ((ulong)digest[5] << 8)
|
||||
| digest[6];
|
||||
ulong key = PakKey.ComposeOpaque(
|
||||
PakAssetType.TexturePayload,
|
||||
payloadId);
|
||||
|
||||
if (_texturePayloadByKey.TryGetValue(key, out TextureIdentity existing))
|
||||
{
|
||||
if (existing.Length != bytes.Length
|
||||
|| !CryptographicOperations.FixedTimeEquals(
|
||||
existing.Sha256,
|
||||
digest))
|
||||
{
|
||||
throw new InvalidDataException(
|
||||
$"texture payload identity collision at key 0x{key:X16}");
|
||||
}
|
||||
|
||||
return key;
|
||||
}
|
||||
|
||||
long offset = _stream.Position;
|
||||
_stream.Write(bytes);
|
||||
PadToAlignment();
|
||||
|
||||
var receipt = new PakTocEntry {
|
||||
Key = key,
|
||||
Offset = (ulong)offset,
|
||||
Length = checked((uint)bytes.Length),
|
||||
Crc32 = Content.Pak.Crc32.Compute(bytes),
|
||||
};
|
||||
_tocEntries.Add(receipt);
|
||||
_receiptByKey.Add(key, receipt);
|
||||
ReserveKey(key);
|
||||
AddReservedBlob(key, bytes);
|
||||
_texturePayloadByKey.Add(
|
||||
key,
|
||||
new TextureIdentity(bytes.Length, digest.ToArray()));
|
||||
TextureBlobCount++;
|
||||
return key;
|
||||
}
|
||||
|
||||
private readonly record struct TextureIdentity(int Length, byte[] Sha256);
|
||||
|
||||
private void ReserveKey(ulong key) {
|
||||
if (!_seenKeys.Add(key)) {
|
||||
throw new ArgumentException($"duplicate pak key 0x{key:X16}", nameof(key));
|
||||
|
|
|
|||
|
|
@ -45,6 +45,10 @@ public static class ContentMigrationCatalog
|
|||
4,
|
||||
5,
|
||||
"solid-colour positive mesh faces must be regenerated"),
|
||||
[6] = FullRebuild(
|
||||
5,
|
||||
6,
|
||||
"the optimized pak v2 texture catalog and compression format require one full rebuild"),
|
||||
};
|
||||
|
||||
public static ContentMigrationPlan Resolve(uint fromRecipeVersion, uint targetRecipeVersion)
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ public sealed record LauncherContentState(
|
|||
public sealed class LauncherContentStateStore
|
||||
{
|
||||
private const uint PakMagic = 0x4B504341u;
|
||||
private const uint PakFormatVersion = 1;
|
||||
private const uint PakFormatVersion = 2;
|
||||
private const int PakHeaderSize = 64;
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
|
|
|
|||
|
|
@ -35,9 +35,9 @@ public sealed record InstallRecordVerification(
|
|||
public sealed class LauncherInstallRecordStore
|
||||
{
|
||||
// Kept in lockstep with AcDream.Content.Pak.PakFormat.CurrentBakeToolVersion
|
||||
// (#426, 2026-08-23: version 5 extracts untextured/solid-colour positive
|
||||
// faces that versions <=4 dropped).
|
||||
public const uint CurrentBakeToolVersion = 5;
|
||||
// Version 6 is pak format 2: global texture payload deduplication plus
|
||||
// adaptive independent-blob compression. It is a mandatory full rebuild.
|
||||
public const uint CurrentBakeToolVersion = 6;
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new()
|
||||
{
|
||||
|
|
|
|||
|
|
@ -106,12 +106,15 @@ public interface ILauncherInstaller
|
|||
/// </summary>
|
||||
public sealed class LauncherInstaller : ILauncherInstaller
|
||||
{
|
||||
public const long FullRebuildRequiredFreeBytes = 2L * 1024 * 1024 * 1024;
|
||||
|
||||
private readonly string _bakeExecutablePath;
|
||||
private readonly DatDirectoryLocator _datDirectories;
|
||||
private readonly LauncherInstallRecordStore _recordStore;
|
||||
private readonly LauncherContentStateStore _contentStateStore;
|
||||
private readonly IBakeProcessRunner _processRunner;
|
||||
private readonly Func<string, CancellationToken, Task<string>> _computeSha256;
|
||||
private readonly Func<string, long> _availableFreeSpace;
|
||||
private readonly SemaphoreSlim _installGate = new(1, 1);
|
||||
|
||||
private LauncherInstallRecord? _verifiedRecord;
|
||||
|
|
@ -128,7 +131,8 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
LauncherInstallRecordStore? recordStore = null,
|
||||
LauncherContentStateStore? contentStateStore = null,
|
||||
IBakeProcessRunner? processRunner = null,
|
||||
Func<string, CancellationToken, Task<string>>? computeSha256 = null)
|
||||
Func<string, CancellationToken, Task<string>>? computeSha256 = null,
|
||||
Func<string, long>? availableFreeSpace = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(paths);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(bakeExecutablePath);
|
||||
|
|
@ -145,6 +149,7 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
_contentStateStore = contentStateStore
|
||||
?? new LauncherContentStateStore(paths, _computeSha256);
|
||||
_processRunner = processRunner ?? new SystemBakeProcessRunner();
|
||||
_availableFreeSpace = availableFreeSpace ?? GetAvailableFreeSpace;
|
||||
}
|
||||
|
||||
public IReadOnlyList<DatDirectoryValidation> DetectDatDirectories() =>
|
||||
|
|
@ -302,10 +307,37 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
?? throw new InvalidOperationException(
|
||||
"The prepared package path has no parent directory."));
|
||||
|
||||
long availableBytes;
|
||||
try
|
||||
{
|
||||
availableBytes = _availableFreeSpace(outputPath);
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException
|
||||
or UnauthorizedAccessException
|
||||
or ArgumentException)
|
||||
{
|
||||
string message =
|
||||
$"Could not check free space for the world-data rebuild: {exception.Message}";
|
||||
Report(progress, LauncherInstallPhase.Failed, message);
|
||||
throw new LauncherInstallException(message, exception);
|
||||
}
|
||||
|
||||
if (availableBytes < FullRebuildRequiredFreeBytes)
|
||||
{
|
||||
string message =
|
||||
$"The optimized world-data rebuild needs at least "
|
||||
+ $"{FullRebuildRequiredFreeBytes / (1024d * 1024d * 1024d):N1} GiB free "
|
||||
+ $"beside the active package; only "
|
||||
+ $"{availableBytes / (1024d * 1024d * 1024d):N1} GiB is available.";
|
||||
Report(progress, LauncherInstallPhase.Failed, message);
|
||||
throw new LauncherInstallException(message);
|
||||
}
|
||||
|
||||
Report(
|
||||
progress,
|
||||
LauncherInstallPhase.PreparingOutput,
|
||||
"Preparing a replacement beside the active package...");
|
||||
"Preparing a one-time optimized world-data rebuild beside the active package. "
|
||||
+ "This can take several minutes; progress will update below...");
|
||||
LauncherInstallRecordStore.TryDelete(bakeOutputPath);
|
||||
LauncherInstallRecordStore.TryDelete(backupPath);
|
||||
bool previousPreserved = false;
|
||||
|
|
@ -1156,6 +1188,16 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
internal static string GetFullRebuildCandidatePath(string outputPath) =>
|
||||
outputPath + ".candidate";
|
||||
|
||||
private static long GetAvailableFreeSpace(string path)
|
||||
{
|
||||
string fullPath = Path.GetFullPath(path);
|
||||
string root = Path.GetPathRoot(fullPath)
|
||||
?? throw new ArgumentException(
|
||||
$"Path '{fullPath}' has no filesystem root.",
|
||||
nameof(path));
|
||||
return new DriveInfo(root).AvailableFreeSpace;
|
||||
}
|
||||
|
||||
private static string BuildChildFailure(
|
||||
int exitCode,
|
||||
string? jsonError,
|
||||
|
|
|
|||
|
|
@ -466,10 +466,9 @@ public sealed class FirstRunInstallerViewModel : ObservableObject, IDisposable
|
|||
string estimate = migration.Kind == ContentWorkKind.Overlay
|
||||
? $"Affected filters: {migration.EffectiveDatIds.Count:N0} DAT id(s), "
|
||||
+ $"{migration.EffectiveLandblocks.Count:N0} landblock(s)."
|
||||
: _contentUpdateBase is { PreparedAssetSize: > 0 } record
|
||||
? $"Free-space guidance: allow about "
|
||||
+ $"{Math.Ceiling(record.PreparedAssetSize * 1.1 / (1024d * 1024d * 1024d)):N0} GiB."
|
||||
: "Free-space guidance: allow room for one complete replacement pak.";
|
||||
: $"Free-space guidance: allow at least "
|
||||
+ $"{LauncherInstaller.FullRebuildRequiredFreeBytes / (1024d * 1024d * 1024d):N0} GiB "
|
||||
+ "while the optimized package is built beside the active one.";
|
||||
return $"This client needs recipe {migration.TargetRecipeVersion}: "
|
||||
+ $"{migration.Reason}. acdream will build {work} from your installed "
|
||||
+ "Asheron's Call DAT files. The existing package stays in place "
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue