From 0dd966f3a086a11e2a84f9bdaf0d446fc12296dc Mon Sep 17 00:00:00 2001 From: Erik Date: Thu, 27 Aug 2026 20:09:09 +0200 Subject: [PATCH] Optimize prepared asset package v2 --- .../2026-08-27-pak-v2-resource-campaign.md | 109 +++++++ src/AcDream.Bake/BakeRunner.cs | 94 +++++- src/AcDream.Content/MeshExtractor.cs | 42 ++- .../Pak/ObjectMeshDataSerializer.cs | 128 ++++++-- src/AcDream.Content/Pak/PakBlobCodec.cs | 124 ++++++++ src/AcDream.Content/Pak/PakFormat.cs | 29 +- src/AcDream.Content/Pak/PakKey.cs | 11 + src/AcDream.Content/Pak/PakReader.cs | 61 +++- .../Pak/PakTexturePayloadCache.cs | 98 ++++++ src/AcDream.Content/Pak/PakWriter.cs | 111 +++++-- .../Installation/ContentMigrationCatalog.cs | 4 + .../Installation/LauncherContentStateStore.cs | 2 +- .../LauncherInstallRecordStore.cs | 6 +- .../Installation/LauncherInstaller.cs | 46 ++- .../ViewModels/FirstRunInstallerViewModel.cs | 7 +- .../BakeDeterminismTests.cs | 2 +- .../InstalledPreparedCollisionCatalogTests.cs | 14 +- .../MeshExtractorSolidFaceExtractionTests.cs | 95 ++++++ .../PakEquivalenceTests.cs | 2 +- .../PakRoundTripTests.cs | 7 +- tests/AcDream.Content.Tests/PakV2Tests.cs | 292 ++++++++++++++++++ .../ContentMigrationCatalogTests.cs | 29 ++ .../LauncherContentStateStoreTests.cs | 2 +- .../Installation/LauncherInstallerTests.cs | 27 ++ .../LauncherOverlayInstallerTests.cs | 24 +- .../LauncherWindowViewModelTests.cs | 1 + tools/run-connected-r6-soak.ps1 | 8 + tools/run-connected-world-lifecycle-gate.ps1 | 8 + 28 files changed, 1277 insertions(+), 106 deletions(-) create mode 100644 docs/plans/2026-08-27-pak-v2-resource-campaign.md create mode 100644 src/AcDream.Content/Pak/PakBlobCodec.cs create mode 100644 src/AcDream.Content/Pak/PakTexturePayloadCache.cs create mode 100644 tests/AcDream.Content.Tests/PakV2Tests.cs create mode 100644 tests/AcDream.Launcher.Core.Tests/Installation/ContentMigrationCatalogTests.cs diff --git a/docs/plans/2026-08-27-pak-v2-resource-campaign.md b/docs/plans/2026-08-27-pak-v2-resource-campaign.md new file mode 100644 index 00000000..c4b08dd0 --- /dev/null +++ b/docs/plans/2026-08-27-pak-v2-resource-campaign.md @@ -0,0 +1,109 @@ +# PAK v2 resource campaign + +Status: ACTIVE (2026-08-27) + +## Objective and release gates + +Ship one crash-safe prepared-asset format migration that: + +- reduces the complete installed package from 29,908,271,024 bytes to at + most 5 GiB; +- preserves decoded geometry, material metadata, texture bytes, deterministic + baking, corruption isolation, and random-access loading; +- does not regress cold or warm world-reveal latency or frame-time percentiles; +- reduces bake CPU/memory and client retained texture memory where the data + permits it, without changing the rendered result; +- gives launcher users a clear one-time update message and progress, while + retaining the last verified package until the replacement is validated; +- passes two different worker-count bakes with identical SHA-256, the complete + installed-DAT bake, content equivalence, performance, solution, Windows CI, + and release gates. + +## Measured format-1 baseline + +The installed package was parsed from its actual TOC, not estimated: + +| Partition | Physical blobs | Physical bytes | +|---|---:|---:| +| GfxObj render meshes | 15,318 | 9,306,115,868 | +| Setup render meshes | 5,935 | 4,078,139 | +| EnvCell render meshes | 17,117 | 20,232,745,510 | +| All collision payloads | 12,938 | 30,392,894 | +| EnvCell topology | 729,888 | 255,512,622 | +| TOC | 2,232,170 rows | 53,572,080 | + +Total: 29,908,271,024 bytes. Render payloads account for approximately +29.54 GB and 99% of physical payload bytes. The collision and index data are +not the size problem. Format 1 already aliases duplicate complete EnvCell +blobs, but each remaining mesh embeds another copy of every decoded RGBA +texture it uses. + +Historical complete-bake baseline: 80.5-107.5 seconds, 4.43-4.89 GB peak +working set, and 3.81-4.21 GB peak private bytes. + +## Format 2 contract + +The 64-byte header and 24-byte sorted TOC row remain fixed. Format version is +2 and bake recipe is 6. + +1. A new `TexturePayload` key partition (type 8) owns globally shared texture + byte arrays. Mesh payloads store the texture payload key while retaining + their own exact dimensions, format, upload metadata, surface identity, + translucency, culling, and index data. +2. Texture payload keys are the first 56 bits of SHA-256 under the type-8 + namespace. The writer retains and compares the full bytes for the complete + bake, making even a truncated-hash collision a loud bake failure rather + than silent substitution. +3. Every physical blob is independently encoded. The high bit of the TOC + length marks compression; the low 31 bits are the stored length. A + compressed blob contains a four-byte decoded-length prefix followed by + Brotli. Small or insufficiently compressible blobs remain exactly raw. + CRC-32 covers stored bytes, then decompression is independently validated. +4. Random access remains one binary search plus one mmap copy for raw blobs. + Compressed blobs add decompression only when the writer proved a material + size win. Texture payloads use a bounded, thread-safe 256 MiB / 2,048-entry + LRU; concurrently decoded meshes converge on one shared array instance. +5. Whole-file compression is forbidden. It would destroy random access and + make a small world reveal depend on unrelated content. + +## Determinism and publication + +Asset traversal and mesh serialization remain sorted. A texture is emitted at +its first deterministic encounter, so its physical order is independent of +worker completion order. Aliases preserve the source row's exact offset, +encoded length/flags, and CRC. + +Recipe 5 to 6 is a mandatory full rebuild. The launcher builds +`acdream.pak.candidate` beside the active package, validates format, recipe, +DAT iterations, TOC counts, size, completion protocol, and SHA-256, then uses +the existing atomic promotion/backup transaction. Cancellation or failure +keeps the verified format-1 package. No overlay may cross this format change. + +## Checkpoint evidence + +The first installed-DAT mixed sample (four GfxObj, three Setup, three EnvCell, +all corresponding collision/topology payloads) produced 58 keys, 29 globally +deduplicated texture payloads, and 57 physical blobs. Decoded payload was +3.5 MiB and stored payload 1.0 MiB (3.62x); output was 1.0 MiB. Eight-worker +and three-worker bakes had the identical SHA-256 +`78886DFA28A3EDF9368A1E25C9B02A3B69ADC5DFCC01358D64A073B549B5B532`. + +This sample is a checkpoint only. The size and performance release gates are +decided by the complete installed-DAT package and connected world-loading +measurements. + +## Work ledger + +- [x] Measure the format-1 package by TOC partition. +- [x] Implement and unit-test format-2 external texture references, adaptive + independent compression, corruption handling, bounded sharing, and byte + determinism. +- [x] Integrate format-2 accounting and strict validation into the bake. +- [x] Publish the recipe-6 mandatory full-rebuild launcher migration. +- [ ] Add launcher disk-space preflight and explicit long-work detail. +- [ ] Complete installed-DAT equivalence and dual-worker full bakes. +- [ ] Measure/tune package size, bake CPU/memory, read CPU/allocations, cold + and warm reveal latency, and frame-time percentiles. +- [ ] Evaluate source-native BC texture retention only if it remains visually + exact and does not shift mip-generation work into the reveal frame. +- [ ] Pass complete tests, Windows CI, merge, push, and release gates. diff --git a/src/AcDream.Bake/BakeRunner.cs b/src/AcDream.Bake/BakeRunner.cs index 6b60a2ea..557c8d13 100644 --- a/src/AcDream.Bake/BakeRunner.cs +++ b/src/AcDream.Bake/BakeRunner.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Diagnostics; using System.Numerics; +using System.Runtime; using AcDream.Content; using AcDream.Content.Pak; using AcDream.Core.Physics; @@ -44,6 +45,7 @@ public sealed record BakeReport public required int UniqueCellStructureCollisions { get; init; } public required int CellStructureCollisionAliases { get; init; } public required int EnvCellTopologyKeys { get; init; } + public required int TexturePayloadKeys { get; init; } public required int SideStagedKeys { get; init; } public int SideStagedDuplicateKeys { get; init; } public required int PhysicalBlobs { get; init; } @@ -54,6 +56,9 @@ public sealed record BakeReport public required long OutputBytes { get; init; } public required long PeakWorkingSetBytes { get; init; } public required long PeakPrivateBytes { get; init; } + public required long DecodedPayloadBytes { get; init; } + public required long StoredPayloadBytes { get; init; } + public required int CompressedBlobs { get; init; } public required IReadOnlyDictionary TypeCounts { get; @@ -62,6 +67,9 @@ public sealed record BakeReport public double EnvCellDedupRatio => UniqueEnvCellGeometries == 0 ? 0 : (double)EnvCellKeys / UniqueEnvCellGeometries; + + public double PayloadCompressionRatio => + StoredPayloadBytes == 0 ? 0 : (double)DecodedPayloadBytes / StoredPayloadBytes; } /// @@ -76,6 +84,7 @@ public static class BakeRunner /// Bounds peak decoded output while keeping workers busy. /// private const int BatchSize = 16; + private const int TransientCollectionStride = 2_048; public static int Run(BakeOptions options) { @@ -183,6 +192,9 @@ public static class BakeRunner ordinaryWork.AddRange( setupIds.Select( id => (PakKey.Compose(PakAssetType.SetupMesh, id), PakAssetType.SetupMesh, id))); + var scheduledMeshKeys = ordinaryWork + .Select(item => item.Key) + .ToHashSet(); ordinaryWork.Sort((a, b) => a.Key.CompareTo(b.Key)); Console.WriteLine( @@ -199,6 +211,7 @@ public static class BakeRunner var header = new PakHeader { + FormatVersion = PakFormat.CurrentFormatVersion, PortalIteration = (uint)dats.Portal.Iteration!.CurrentIteration, CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration, HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration, @@ -272,7 +285,11 @@ public static class BakeRunner sideStagedDuped += DrainSideStaged( sideStaged, sideStagedByKey, - writtenKeys); + writtenKeys, + scheduledMeshKeys); + CompactTransientBakeBuffersIfDue( + batchStart + batch.Length, + final: batchStart + BatchSize >= ordinaryWork.Count); ReportProgressIfDue( completed, totalExtractionJobs, @@ -373,7 +390,11 @@ public static class BakeRunner sideStagedDuped += DrainSideStaged( sideStaged, sideStagedByKey, - writtenKeys); + writtenKeys, + scheduledMeshKeys); + CompactTransientBakeBuffersIfDue( + batchStart + batch.Length, + final: batchStart + BatchSize >= envCatalog.Groups.Count); ReportProgressIfDue( completed, totalExtractionJobs, @@ -801,6 +822,7 @@ public static class BakeRunner [PakAssetType.CellStructureCollision] = cellStructureWritten, [PakAssetType.EnvCellTopology] = envCellTopologyWritten, + [PakAssetType.TexturePayload] = writer.TextureBlobCount, }; var report = new BakeReport { @@ -821,23 +843,19 @@ public static class BakeRunner CellStructureCollisionAliases = cellStructureAliasCount, EnvCellTopologyKeys = envCellTopologyWritten, + TexturePayloadKeys = writer.TextureBlobCount, SideStagedKeys = sideStagedWritten, - PhysicalBlobs = - gfxObjWritten - + setupWritten - + envCellGeometriesWritten - + sideStagedWritten - + uniqueGfxCollisions - + uniqueSetupCollisions - + uniqueCellStructures - + envCellTopologyWritten, - TotalKeys = writtenKeys.Count, + PhysicalBlobs = writer.PhysicalBlobCount, + TotalKeys = writer.EntryCount, Failures = failures.Count, ExtractionAndWriteElapsed = stopwatch.Elapsed, Elapsed = stopwatch.Elapsed, OutputBytes = new FileInfo(options.OutPath).Length, PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64, PeakPrivateBytes = Process.GetCurrentProcess().PeakPagedMemorySize64, + DecodedPayloadBytes = writer.DecodedPayloadBytes, + StoredPayloadBytes = writer.StoredPayloadBytes, + CompressedBlobs = writer.CompressedBlobCount, TypeCounts = typeCounts, }; @@ -907,19 +925,60 @@ public static class BakeRunner private static int DrainSideStaged( ConcurrentQueue sideStaged, Dictionary sideStagedByKey, - HashSet writtenKeys) + HashSet writtenKeys, + HashSet scheduledMeshKeys) { int duplicates = 0; while (sideStaged.TryDequeue(out var staged)) { uint fileId = (uint)(staged.ObjectId & 0xFFFF_FFFFu); ulong key = PakKey.Compose(PakAssetType.GfxObjMesh, fileId); - if (writtenKeys.Contains(key) || !sideStagedByKey.TryAdd(key, staged)) + // A full bake already schedules every GfxObj. Retaining an early + // particle-preload copy until that later row was written pinned + // thousands of complete meshes and several GiB of texture arrays. + // Only filtered bakes need to retain a side-staged key that is not + // otherwise in their work list. + if (writtenKeys.Contains(key) + || scheduledMeshKeys.Contains(key) + || !sideStagedByKey.TryAdd(key, staged)) duplicates++; } return duplicates; } + /// + /// Mesh decode and compression use large temporary pixel/serialization + /// arrays. A short offline bake otherwise finishes before workstation GC + /// decides to compact the LOH, leaving several GiB committed despite the + /// live set being bounded. Batch boundaries have no active workers and + /// are the deterministic safe point to reclaim those transients. The + /// stride keeps the CPU cost small while bounding peak memory. + /// + private static void CompactTransientBakeBuffersIfDue( + int completedItems, + bool final) + { + if (!final && completedItems % TransientCollectionStride != 0) + return; + + GCLargeObjectHeapCompactionMode previous = + GCSettings.LargeObjectHeapCompactionMode; + try + { + GCSettings.LargeObjectHeapCompactionMode = + GCLargeObjectHeapCompactionMode.CompactOnce; + GC.Collect( + GC.MaxGeneration, + GCCollectionMode.Forced, + blocking: true, + compacting: true); + } + finally + { + GCSettings.LargeObjectHeapCompactionMode = previous; + } + } + private static void ReportProgressIfDue( long done, int total, @@ -977,11 +1036,18 @@ public static class BakeRunner $"{report.CellStructureCollisionAliases:N0} aliases"); Console.WriteLine( $" Cell topologies: {report.EnvCellTopologyKeys:N0}"); + Console.WriteLine( + $" texture payloads: {report.TexturePayloadKeys:N0}"); Console.WriteLine( $" side-staged keys: {report.SideStagedKeys:N0} " + $"({report.SideStagedDuplicateKeys:N0} duplicate keys)"); Console.WriteLine($" physical blobs: {report.PhysicalBlobs:N0}"); Console.WriteLine($" total keys: {report.TotalKeys:N0}"); + Console.WriteLine( + $" compressed blobs: {report.CompressedBlobs:N0}; " + + $"payload {report.DecodedPayloadBytes / 1024.0 / 1024.0:F1} -> " + + $"{report.StoredPayloadBytes / 1024.0 / 1024.0:F1} MB " + + $"({report.PayloadCompressionRatio:F2}x)"); Console.WriteLine($" failures: {report.Failures:N0}"); Console.WriteLine( $" extract + write: {report.ExtractionAndWriteElapsed.TotalSeconds:F1} s"); diff --git a/src/AcDream.Content/MeshExtractor.cs b/src/AcDream.Content/MeshExtractor.cs index f3cba889..0791423d 100644 --- a/src/AcDream.Content/MeshExtractor.cs +++ b/src/AcDream.Content/MeshExtractor.cs @@ -433,7 +433,18 @@ public sealed class MeshExtractor { isClipMap, surface.Type.HasFlag(SurfaceType.Additive)); - if (TextureHelpers.IsCompressedFormat(renderSurface.Format)) { + if (CanPreserveCompressedTexture( + renderSurface.Format, + isClipMap, + surface.Translucency)) { + // Vulkan requires textureCompressionBC, so an unedited + // DAT DXT surface can stay byte-exact all the way to the + // GPU. Decoding it here inflated pak, CPU, and GPU + // residency by up to eight times. + textureFormat = ToTextureFormat(renderSurface.Format); + textureData = renderSurface.SourceData; + } + else if (TextureHelpers.IsCompressedFormat(renderSurface.Format)) { isDxt3or5 = renderSurface.Format == DatReaderWriter.Enums.PixelFormat.PFID_DXT3 || renderSurface.Format == DatReaderWriter.Enums.PixelFormat.PFID_DXT5; textureFormat = TextureFormat.RGBA8; uploadPixelFormat = UploadPixelFormat.Rgba; @@ -822,7 +833,14 @@ public sealed class MeshExtractor { isClipMap, surface.Type.HasFlag(SurfaceType.Additive)); - if (_decodedTextureCache.TryGet(decodedTextureKey, out var cachedData)) { + if (CanPreserveCompressedTexture( + renderSurface.Format, + isClipMap, + surface.Translucency)) { + textureData = renderSurface.SourceData; + textureFormat = ToTextureFormat(renderSurface.Format); + } + else if (_decodedTextureCache.TryGet(decodedTextureKey, out var cachedData)) { textureData = cachedData; textureFormat = TextureFormat.RGBA8; uploadPixelFormat = UploadPixelFormat.Rgba; @@ -1126,6 +1144,26 @@ public sealed class MeshExtractor { return textureData; } + private static bool CanPreserveCompressedTexture( + DatReaderWriter.Enums.PixelFormat format, + bool isClipMap, + float translucency) => + TextureHelpers.IsCompressedFormat(format) + && !isClipMap + && translucency <= 0.0f; + + private static TextureFormat ToTextureFormat( + DatReaderWriter.Enums.PixelFormat format) => format switch + { + DatReaderWriter.Enums.PixelFormat.PFID_DXT1 => TextureFormat.DXT1, + DatReaderWriter.Enums.PixelFormat.PFID_DXT3 => TextureFormat.DXT3, + DatReaderWriter.Enums.PixelFormat.PFID_DXT5 => TextureFormat.DXT5, + _ => throw new ArgumentOutOfRangeException( + nameof(format), + format, + "The source is not a supported block-compressed texture."), + }; + private void BuildPolygonIndices(Polygon poly, GfxObj gfxObj, Vector3 scale, Dictionary<(ushort vertId, ushort uvIdx, bool isNeg), ushort> UVLookup, List vertices, List indices, bool useNegSurface, ref bool hasWrappingUVs) { diff --git a/src/AcDream.Content/Pak/ObjectMeshDataSerializer.cs b/src/AcDream.Content/Pak/ObjectMeshDataSerializer.cs index 0fe0732d..277b0a70 100644 --- a/src/AcDream.Content/Pak/ObjectMeshDataSerializer.cs +++ b/src/AcDream.Content/Pak/ObjectMeshDataSerializer.cs @@ -36,34 +36,63 @@ namespace AcDream.Content.Pak; public static class ObjectMeshDataSerializer { public static void Write(ObjectMeshData data, Stream stream) { using var bw = new BinaryWriter(stream, System.Text.Encoding.UTF8, leaveOpen: true); - WriteObjectMeshData(bw, data); + WriteObjectMeshData(bw, data, externalTextures: null); + } + + public static void WriteExternalTextures( + ObjectMeshData data, + Stream stream, + Func registerTexture) { + ArgumentNullException.ThrowIfNull(registerTexture); + using var bw = new BinaryWriter( + stream, + System.Text.Encoding.UTF8, + leaveOpen: true); + WriteObjectMeshData(bw, data, registerTexture); } /// Deserializes directly over — no defensive copy (PakReader's single-pass read reuses its CRC buffer here). public static ObjectMeshData Read(byte[] bytes) { using var ms = new MemoryStream(bytes, writable: false); using var br = new BinaryReader(ms, System.Text.Encoding.UTF8, leaveOpen: true); - return ReadObjectMeshData(br); + return ReadObjectMeshData(br, externalTextures: null); } public static ObjectMeshData Read(ReadOnlySpan bytes) => Read(bytes.ToArray()); + public static ObjectMeshData ReadExternalTextures( + byte[] bytes, + Func resolveTexture) { + ArgumentNullException.ThrowIfNull(resolveTexture); + using var ms = new MemoryStream(bytes, writable: false); + using var br = new BinaryReader( + ms, + System.Text.Encoding.UTF8, + leaveOpen: true); + return ReadObjectMeshData(br, resolveTexture); + } + // ---- ObjectMeshData ----------------------------------------------------- - private static void WriteObjectMeshData(BinaryWriter w, ObjectMeshData data) { + private static void WriteObjectMeshData( + BinaryWriter w, + ObjectMeshData data, + Func? externalTextures) { w.Write(data.ObjectId); w.Write(data.IsSetup); WriteVertexArray(w, data.Vertices); w.Write(data.Batches.Count); - foreach (var batch in data.Batches) WriteMeshBatchData(w, batch); + foreach (var batch in data.Batches) + WriteMeshBatchData(w, batch, externalTextures); w.Write(data.UploadAttempts); // EnvCellGeometry: recursive nested block. w.Write(data.EnvCellGeometry is not null); - if (data.EnvCellGeometry is not null) WriteObjectMeshData(w, data.EnvCellGeometry); + if (data.EnvCellGeometry is not null) + WriteObjectMeshData(w, data.EnvCellGeometry, externalTextures); w.Write(data.SetupParts.Count); foreach (var (gfxObjId, transform) in data.SetupParts) { @@ -74,7 +103,7 @@ public static class ObjectMeshDataSerializer { w.Write(data.ParticleEmitters.Count); foreach (var emitter in data.ParticleEmitters) WriteStagedEmitter(w, emitter); - WriteTextureBatches(w, data.TextureBatches); + WriteTextureBatches(w, data.TextureBatches, externalTextures); WriteBoundingBox(w, data.BoundingBox); WriteVector3(w, data.SortCenter); @@ -86,7 +115,9 @@ public static class ObjectMeshDataSerializer { WriteVector3Array(w, data.EdgeLines); } - private static ObjectMeshData ReadObjectMeshData(BinaryReader r) { + private static ObjectMeshData ReadObjectMeshData( + BinaryReader r, + Func? externalTextures) { var data = new ObjectMeshData { ObjectId = r.ReadUInt64(), IsSetup = r.ReadBoolean(), @@ -96,13 +127,16 @@ public static class ObjectMeshDataSerializer { int batchCount = r.ReadInt32(); var batches = new List(batchCount); - for (int i = 0; i < batchCount; i++) batches.Add(ReadMeshBatchData(r)); + for (int i = 0; i < batchCount; i++) + batches.Add(ReadMeshBatchData(r, externalTextures)); data.Batches = batches; data.UploadAttempts = r.ReadInt32(); bool hasEnvCellGeometry = r.ReadBoolean(); - data.EnvCellGeometry = hasEnvCellGeometry ? ReadObjectMeshData(r) : null; + data.EnvCellGeometry = hasEnvCellGeometry + ? ReadObjectMeshData(r, externalTextures) + : null; int setupPartCount = r.ReadInt32(); var setupParts = new List<(ulong GfxObjId, Matrix4x4 Transform)>(setupPartCount); @@ -118,7 +152,7 @@ public static class ObjectMeshDataSerializer { for (int i = 0; i < emitterCount; i++) emitters.Add(ReadStagedEmitter(r)); data.ParticleEmitters = emitters; - data.TextureBatches = ReadTextureBatches(r); + data.TextureBatches = ReadTextureBatches(r, externalTextures); data.BoundingBox = ReadBoundingBox(r); data.SortCenter = ReadVector3(r); @@ -134,24 +168,37 @@ public static class ObjectMeshDataSerializer { // ---- MeshBatchData ------------------------------------------------------- - private static void WriteMeshBatchData(BinaryWriter w, MeshBatchData batch) { + private static void WriteMeshBatchData( + BinaryWriter w, + MeshBatchData batch, + Func? externalTextures) { WriteUInt16Array(w, batch.Indices); WriteTextureFormatTuple(w, batch.TextureFormat); WriteTextureKey(w, batch.TextureKey); w.Write(batch.TextureIndex); - WriteByteArray(w, batch.TextureData); + WriteTexturePayload( + w, + batch.TextureKey, + batch.TextureData, + externalTextures); WriteNullableInt32Enum(w, batch.UploadPixelFormat.HasValue, batch.UploadPixelFormat is { } upf ? (int)upf : 0); WriteNullableInt32Enum(w, batch.UploadPixelType.HasValue, batch.UploadPixelType is { } upt ? (int)upt : 0); w.Write((int)batch.CullMode); } - private static MeshBatchData ReadMeshBatchData(BinaryReader r) { + private static MeshBatchData ReadMeshBatchData( + BinaryReader r, + Func? externalTextures) { + ushort[] indices = ReadUInt16Array(r); + (int Width, int Height, TextureFormat Format) format = + ReadTextureFormatTuple(r); + TextureKey key = ReadTextureKey(r); var batch = new MeshBatchData { - Indices = ReadUInt16Array(r), - TextureFormat = ReadTextureFormatTuple(r), - TextureKey = ReadTextureKey(r), + Indices = indices, + TextureFormat = format, + TextureKey = key, TextureIndex = r.ReadInt32(), - TextureData = ReadByteArray(r), + TextureData = ReadTexturePayload(r, externalTextures), }; batch.UploadPixelFormat = ReadNullableInt32Enum(r, v => (UploadPixelFormat)v); batch.UploadPixelType = ReadNullableInt32Enum(r, v => (UploadPixelType)v); @@ -161,9 +208,12 @@ public static class ObjectMeshDataSerializer { // ---- TextureBatchData / TextureBatches dictionary ------------------------- - private static void WriteTextureBatchData(BinaryWriter w, TextureBatchData batch) { + private static void WriteTextureBatchData( + BinaryWriter w, + TextureBatchData batch, + Func? externalTextures) { WriteTextureKey(w, batch.Key); - WriteByteArray(w, batch.TextureData); + WriteTexturePayload(w, batch.Key, batch.TextureData, externalTextures); WriteNullableInt32Enum(w, batch.UploadPixelFormat.HasValue, batch.UploadPixelFormat is { } upf ? (int)upf : 0); WriteNullableInt32Enum(w, batch.UploadPixelType.HasValue, batch.UploadPixelType is { } upt ? (int)upt : 0); WriteUInt16List(w, batch.Indices); @@ -174,10 +224,13 @@ public static class ObjectMeshDataSerializer { w.Write(batch.HasWrappingUVs); } - private static TextureBatchData ReadTextureBatchData(BinaryReader r) { + private static TextureBatchData ReadTextureBatchData( + BinaryReader r, + Func? externalTextures) { + TextureKey key = ReadTextureKey(r); var batch = new TextureBatchData { - Key = ReadTextureKey(r), - TextureData = ReadByteArray(r), + Key = key, + TextureData = ReadTexturePayload(r, externalTextures), }; batch.UploadPixelFormat = ReadNullableInt32Enum(r, v => (UploadPixelFormat)v); batch.UploadPixelType = ReadNullableInt32Enum(r, v => (UploadPixelType)v); @@ -198,7 +251,8 @@ public static class ObjectMeshDataSerializer { /// private static void WriteTextureBatches( BinaryWriter w, - Dictionary<(int Width, int Height, TextureFormat Format), List> batches) { + Dictionary<(int Width, int Height, TextureFormat Format), List> batches, + Func? externalTextures) { var sortedKeys = batches.Keys .OrderBy(k => k.Width) .ThenBy(k => k.Height) @@ -213,11 +267,14 @@ public static class ObjectMeshDataSerializer { var list = batches[key]; w.Write(list.Count); - foreach (var item in list) WriteTextureBatchData(w, item); + foreach (var item in list) + WriteTextureBatchData(w, item, externalTextures); } } - private static Dictionary<(int Width, int Height, TextureFormat Format), List> ReadTextureBatches(BinaryReader r) { + private static Dictionary<(int Width, int Height, TextureFormat Format), List> ReadTextureBatches( + BinaryReader r, + Func? externalTextures) { int groupCount = r.ReadInt32(); var result = new Dictionary<(int Width, int Height, TextureFormat Format), List>(groupCount); for (int i = 0; i < groupCount; i++) { @@ -227,13 +284,32 @@ public static class ObjectMeshDataSerializer { int listCount = r.ReadInt32(); var list = new List(listCount); - for (int j = 0; j < listCount; j++) list.Add(ReadTextureBatchData(r)); + for (int j = 0; j < listCount; j++) + list.Add(ReadTextureBatchData(r, externalTextures)); result[(width, height, format)] = list; } return result; } + private static void WriteTexturePayload( + BinaryWriter writer, + TextureKey key, + byte[] bytes, + Func? externalTextures) { + if (externalTextures is null) + WriteByteArray(writer, bytes); + else + writer.Write(externalTextures(key, bytes)); + } + + private static byte[] ReadTexturePayload( + BinaryReader reader, + Func? externalTextures) => + externalTextures is null + ? ReadByteArray(reader) + : externalTextures(reader.ReadUInt64()); + // ---- StagedEmitter / ParticleEmitter (DBObj) ------------------------------ private static void WriteStagedEmitter(BinaryWriter w, StagedEmitter emitter) { diff --git a/src/AcDream.Content/Pak/PakBlobCodec.cs b/src/AcDream.Content/Pak/PakBlobCodec.cs new file mode 100644 index 00000000..ee420924 --- /dev/null +++ b/src/AcDream.Content/Pak/PakBlobCodec.cs @@ -0,0 +1,124 @@ +using System.Buffers; +using System.Buffers.Binary; +using System.IO.Compression; + +namespace AcDream.Content.Pak; + +/// +/// Format-2 independent-blob compression. Each payload remains separately +/// addressable; incompressible and small payloads stay raw so a size win can +/// never turn into needless runtime CPU work. +/// +internal static class PakBlobCodec +{ + // The largest format-1 payload in the complete installed package is + // 19,683,674 bytes. Texture externalization makes format-2 mesh blobs + // smaller still. A fixed 64 MiB ceiling prevents a corrupt package from + // declaring a multi-gigabyte allocation before Brotli can reject it. + internal const int MaximumDecodedBytes = 64 * 1024 * 1024; + private const int DecodedLengthPrefixSize = sizeof(uint); + private const int MinimumCompressionBytes = 512; + private const int MinimumSavingsBytes = 64; + private const int BrotliQuality = 1; + private const int BrotliWindow = 22; + + internal readonly record struct Encoded(byte[] Bytes, bool Compressed); + + /// + /// Attempts a worthwhile compression into an ArrayPool-owned buffer. + /// The caller must return a non-null buffer. A false result owns no + /// buffer and should write the original bytes directly. + /// + public static bool TryCompress( + ReadOnlySpan decoded, + out byte[]? rented, + out int storedLength) + { + if (decoded.Length > MaximumDecodedBytes) + throw new InvalidDataException( + $"pak payload is {decoded.Length} bytes; maximum is {MaximumDecodedBytes}"); + + rented = null; + storedLength = decoded.Length; + if (decoded.Length < MinimumCompressionBytes) + return false; + + int maximum = checked( + DecodedLengthPrefixSize + + BrotliEncoder.GetMaxCompressedLength(decoded.Length)); + byte[] candidate = ArrayPool.Shared.Rent(maximum); + if (!BrotliEncoder.TryCompress( + decoded, + candidate.AsSpan(DecodedLengthPrefixSize, maximum - DecodedLengthPrefixSize), + out int compressedLength, + BrotliQuality, + BrotliWindow)) + { + ArrayPool.Shared.Return(candidate); + return false; + } + + storedLength = checked(DecodedLengthPrefixSize + compressedLength); + int requiredSavings = Math.Max( + MinimumSavingsBytes, + decoded.Length / 16); + if (decoded.Length - storedLength < requiredSavings) + { + ArrayPool.Shared.Return(candidate); + storedLength = decoded.Length; + return false; + } + + BinaryPrimitives.WriteUInt32LittleEndian( + candidate.AsSpan(0, DecodedLengthPrefixSize), + checked((uint)decoded.Length)); + rented = candidate; + return true; + } + + // Test/convenience wrapper. Production PakWriter uses TryCompress so it + // can write the pooled buffer directly without a compacting copy. + public static Encoded Encode(byte[] decoded) + { + ArgumentNullException.ThrowIfNull(decoded); + if (!TryCompress(decoded, out byte[]? rented, out int storedLength)) + return new Encoded(decoded, false); + + byte[] buffer = rented + ?? throw new InvalidOperationException("compressed buffer was not returned"); + try + { + return new Encoded(buffer.AsSpan(0, storedLength).ToArray(), true); + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + public static byte[] Decode(ReadOnlySpan stored) + { + if (stored.Length < DecodedLengthPrefixSize) + throw new InvalidDataException( + "compressed pak blob is missing its decoded-length prefix"); + + uint decodedLength = BinaryPrimitives.ReadUInt32LittleEndian( + stored[..DecodedLengthPrefixSize]); + if (decodedLength > MaximumDecodedBytes) + throw new InvalidDataException( + $"compressed pak blob declares unsupported decoded length {decodedLength}"); + + var decoded = new byte[checked((int)decodedLength)]; + if (!BrotliDecoder.TryDecompress( + stored[DecodedLengthPrefixSize..], + decoded, + out int written) + || written != decoded.Length) + { + throw new InvalidDataException( + $"compressed pak blob did not decode to its declared {decoded.Length} bytes"); + } + + return decoded; + } +} diff --git a/src/AcDream.Content/Pak/PakFormat.cs b/src/AcDream.Content/Pak/PakFormat.cs index 590b7e6a..2256ced5 100644 --- a/src/AcDream.Content/Pak/PakFormat.cs +++ b/src/AcDream.Content/Pak/PakFormat.cs @@ -14,7 +14,7 @@ public static class PakFormat { /// template); PakReader refuses to open any other version. Bump ONLY /// with an accompanying reader migration path. /// - public const uint CurrentFormatVersion = 1; + public const uint CurrentFormatVersion = 2; /// /// Identity of the bake algorithm that produced the payloads. Version 2 @@ -28,9 +28,11 @@ public static class PakFormat { /// zero vertices on its positive side, so any pak baked by an older tool /// is missing those faces (e.g. the Holtburg windmill axle 0x010010CE, /// 8 polygons all NoPos + Base1Solid, extracted to a 0-vertex mesh). The - /// binary format remains version 1. + /// binary format remains version 1. Version 6 introduces pak format 2: + /// globally shared texture payloads plus independently Brotli-compressed + /// blobs with raw fallback. Mesh geometry and decoded pixels remain exact. /// - public const uint CurrentBakeToolVersion = 5; + public const uint CurrentBakeToolVersion = 6; } /// @@ -38,7 +40,7 @@ public static class PakFormat { /// /// offset size field /// 0 4 magic 'ACPK' (0x4B504341) -/// 4 4 formatVersion = 1 +/// 4 4 formatVersion = 2 /// 8 4 portalIteration (DatCollection.Portal.Iteration) /// 12 4 cellIteration /// 16 4 highResIteration @@ -48,7 +50,9 @@ public static class PakFormat { /// 36 4 bakeToolVersion /// 40 24 reserved (zero) /// -/// Spec: docs/superpowers/plans/2026-07-05-mp1b-pak-and-bake.md "Format v1 (normative)". +/// Format 1 is specified by docs/superpowers/plans/2026-07-05-mp1b-pak-and-bake.md. +/// Format 2 retains this header and adds the TOC compression flag plus global +/// texture payload entries documented in docs/plans/2026-08-27-pak-v2-resource-campaign.md. /// public struct PakHeader { public const int Size = 64; @@ -114,19 +118,32 @@ public struct PakHeader { } /// -/// One 24-byte TOC entry: key u64, offset u64, length u32, crc32 u32. +/// One 24-byte TOC entry: key u64, offset u64, lengthAndFlags u32, crc32 u32. /// Entries in a pak's TOC are sorted ascending by to allow /// binary-search lookup. is a corruption tripwire computed /// over the blob bytes; the reader verifies lazily on first access. /// public struct PakTocEntry { public const int Size = 24; + public const uint CompressionFlag = 0x8000_0000u; + public const uint StoredLengthMask = 0x7FFF_FFFFu; public ulong Key; public ulong Offset; public uint Length; public uint Crc32; + /// + /// Format-2 entries use the high bit of to mark an + /// independently Brotli-compressed blob. The remaining 31 bits are the + /// exact bytes stored in the file (including the compressed blob's + /// four-byte decoded-length prefix). Raw entries retain their historical + /// length representation and require no second copy on read. + /// + public readonly bool IsCompressed => (Length & CompressionFlag) != 0; + + public readonly uint StoredLength => Length & StoredLengthMask; + public void WriteTo(Span dest) { if (dest.Length < Size) throw new ArgumentException($"destination must be at least {Size} bytes", nameof(dest)); BinaryPrimitives.WriteUInt64LittleEndian(dest[0..8], Key); diff --git a/src/AcDream.Content/Pak/PakKey.cs b/src/AcDream.Content/Pak/PakKey.cs index 8d98d460..72a5931e 100644 --- a/src/AcDream.Content/Pak/PakKey.cs +++ b/src/AcDream.Content/Pak/PakKey.cs @@ -15,6 +15,7 @@ public enum PakAssetType : byte { SetupCollision = 5, CellStructureCollision = 6, EnvCellTopology = 7, + TexturePayload = 8, } /// @@ -34,4 +35,14 @@ public static class PakKey { var fileId = (uint)((key >> 24) & 0xFFFFFFFFu); return (type, fileId); } + + /// + /// Composes a content-addressed format-2 key whose lower 56 bits are an + /// opaque payload identity rather than the legacy file-id/reserved split. + /// + public static ulong ComposeOpaque(PakAssetType type, ulong payloadId) { + if ((payloadId & 0xFF00_0000_0000_0000ul) != 0) + throw new ArgumentOutOfRangeException(nameof(payloadId)); + return ((ulong)type << 56) | payloadId; + } } diff --git a/src/AcDream.Content/Pak/PakReader.cs b/src/AcDream.Content/Pak/PakReader.cs index c6767e44..adf84696 100644 --- a/src/AcDream.Content/Pak/PakReader.cs +++ b/src/AcDream.Content/Pak/PakReader.cs @@ -40,6 +40,7 @@ public sealed class PakReader : IDisposable { private readonly MemoryMappedViewAccessor _accessor; private readonly long _fileLength; private readonly PakTocEntry[] _toc; // sorted ascending by Key + private readonly PakTexturePayloadCache _texturePayloads = new(); /// Lazy per-entry verdict: absent = not yet judged, 0 = bad (bounds/crc/structure), 1 = ok. private readonly ConcurrentDictionary _entryVerdictByTocIndex = new(); @@ -100,7 +101,7 @@ public sealed class PakReader : IDisposable { bool invalid = !IsEntryRangeValid(entry); if (invalid) { _entryVerdictByTocIndex[i] = 0; - LogCorruptionOnce(i, $"TOC entry out of bounds (offset={entry.Offset}, length={entry.Length}, " + + LogCorruptionOnce(i, $"TOC entry out of bounds (offset={entry.Offset}, storedLength={entry.StoredLength}, " + $"file={_fileLength}, toc@{Header.TocOffset})"); } } @@ -158,7 +159,9 @@ public sealed class PakReader : IDisposable { return status; try { - data = ObjectMeshDataSerializer.Read(bytes); + data = ObjectMeshDataSerializer.ReadExternalTextures( + bytes, + ResolveTexturePayload); return PakObjectReadStatus.Loaded; } catch (Exception ex) { @@ -194,8 +197,9 @@ public sealed class PakReader : IDisposable { // Single pass (review finding 4): ONE copy out of the map; CRC and // deserialization both run over this same buffer. ref readonly var entry = ref _toc[index]; - bytes = new byte[entry.Length]; - _accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length); + uint storedLength = entry.StoredLength; + bytes = new byte[checked((int)storedLength)]; + _accessor.ReadArray((long)entry.Offset, bytes, 0, checked((int)storedLength)); if (!judged) { uint actualCrc = Crc32.Compute(bytes); @@ -208,9 +212,41 @@ public sealed class PakReader : IDisposable { _entryVerdictByTocIndex[index] = 1; } + if (entry.IsCompressed) { + try { + bytes = PakBlobCodec.Decode(bytes); + } + catch (Exception ex) when (ex is InvalidDataException or OverflowException) { + _entryVerdictByTocIndex[index] = 0; + LogCorruptionOnce( + index, + $"decompression failed despite matching CRC: {ex.Message}"); + bytes = null; + return PakObjectReadStatus.Corrupt; + } + } + return PakObjectReadStatus.Loaded; } + private byte[] ResolveTexturePayload(ulong key) { + if (PakKey.Decompose(key).Type != PakAssetType.TexturePayload) { + throw new InvalidDataException( + $"mesh references non-texture pak key 0x{key:X16}"); + } + + if (_texturePayloads.TryGet(key, out byte[] cachedBytes)) + return cachedBytes; + + PakObjectReadStatus status = ReadBlobBytes(key, out byte[]? loadedBytes); + if (status != PakObjectReadStatus.Loaded || loadedBytes is null) { + throw new InvalidDataException( + $"mesh references {status.ToString().ToLowerInvariant()} texture payload 0x{key:X16}"); + } + + return _texturePayloads.AddOrGet(key, loadedBytes); + } + internal void MarkPayloadCorrupt(ulong key, string reason) { int index = BinarySearch(key); if (index < 0) @@ -269,7 +305,7 @@ public sealed class PakReader : IDisposable { if (!IsEntryRangeValid(entry)) { throw new InvalidDataException( $"pak TOC entry 0x{entry.Key:X16} has an invalid range " + - $"(offset={entry.Offset}, length={entry.Length}, " + + $"(offset={entry.Offset}, storedLength={entry.StoredLength}, " + $"file={_fileLength}, toc@{Header.TocOffset})"); } } @@ -288,8 +324,9 @@ public sealed class PakReader : IDisposable { if (_entryVerdictByTocIndex.TryGetValue(tocIndex, out var cached)) return cached; ref readonly var entry = ref _toc[tocIndex]; - var bytes = new byte[entry.Length]; - _accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length); + uint storedLength = entry.StoredLength; + var bytes = new byte[checked((int)storedLength)]; + _accessor.ReadArray((long)entry.Offset, bytes, 0, checked((int)storedLength)); uint actualCrc = Crc32.Compute(bytes); bool ok = actualCrc == entry.Crc32; @@ -304,7 +341,8 @@ public sealed class PakReader : IDisposable { if (!_loggedCorruption.TryAdd(tocIndex, true)) return; ref readonly var entry = ref _toc[tocIndex]; Console.Error.WriteLine( - $"[pak-corrupt] key 0x{entry.Key:X16} at offset {entry.Offset} (length {entry.Length}): " + + $"[pak-corrupt] key 0x{entry.Key:X16} at offset {entry.Offset} " + + $"(storedLength {entry.StoredLength}, compressed={entry.IsCompressed}): " + $"{reason} — treating as missing"); } @@ -329,8 +367,11 @@ public sealed class PakReader : IDisposable { return false; } - return entry.Length <= Header.TocOffset - entry.Offset && - entry.Length <= fileLength - entry.Offset; + if (entry.StoredLength > PakBlobCodec.MaximumDecodedBytes) + return false; + + return entry.StoredLength <= Header.TocOffset - entry.Offset && + entry.StoredLength <= fileLength - entry.Offset; } public void Dispose() { diff --git a/src/AcDream.Content/Pak/PakTexturePayloadCache.cs b/src/AcDream.Content/Pak/PakTexturePayloadCache.cs new file mode 100644 index 00000000..328bfcaf --- /dev/null +++ b/src/AcDream.Content/Pak/PakTexturePayloadCache.cs @@ -0,0 +1,98 @@ +namespace AcDream.Content.Pak; + +/// +/// Bounded shared-byte owner for format-2 texture payloads. Mesh objects keep +/// ordinary array references, so evicting a cache row is safe while preventing +/// a long travel session from pinning every texture ever observed. +/// +internal sealed class PakTexturePayloadCache +{ + // Payloads are staging data on the way to GPU atlases. A 256 MiB cache + // could retain a second world-sized texture working set after the atlas + // already owned it; native BC makes 64 MiB ample for revisit sharing. + internal const long DefaultMaximumBytes = 64L * 1024 * 1024; + internal const int DefaultMaximumEntries = 1_024; + + private readonly long _maximumBytes; + private readonly int _maximumEntries; + private readonly Dictionary _entries = []; + private readonly LinkedList _lru = []; + private readonly Lock _gate = new(); + private long _bytes; + + internal int Count + { + get { lock (_gate) return _entries.Count; } + } + + internal long Bytes + { + get { lock (_gate) return _bytes; } + } + + private sealed record Entry(byte[] Bytes, LinkedListNode Node); + + public PakTexturePayloadCache( + long maximumBytes = DefaultMaximumBytes, + int maximumEntries = DefaultMaximumEntries) + { + ArgumentOutOfRangeException.ThrowIfLessThan(maximumBytes, 1); + ArgumentOutOfRangeException.ThrowIfLessThan(maximumEntries, 1); + _maximumBytes = maximumBytes; + _maximumEntries = maximumEntries; + } + + public bool TryGet(ulong key, out byte[] bytes) + { + lock (_gate) + { + if (!_entries.TryGetValue(key, out Entry? entry)) + { + bytes = null!; + return false; + } + + _lru.Remove(entry.Node); + _lru.AddLast(entry.Node); + bytes = entry.Bytes; + return true; + } + } + + public byte[] AddOrGet(ulong key, byte[] bytes) + { + ArgumentNullException.ThrowIfNull(bytes); + lock (_gate) + { + if (_entries.TryGetValue(key, out Entry? existing)) + { + _lru.Remove(existing.Node); + _lru.AddLast(existing.Node); + return existing.Bytes; + } + + // A single unusually large texture must not defeat the global + // bound. Its caller still owns the returned array; it simply is + // not retained for a later mesh. + if (bytes.LongLength > _maximumBytes) + return bytes; + + LinkedListNode node = _lru.AddLast(key); + _entries.Add(key, new Entry(bytes, node)); + _bytes = checked(_bytes + bytes.LongLength); + Trim(); + return bytes; + } + } + + private void Trim() + { + while ((_bytes > _maximumBytes || _entries.Count > _maximumEntries) + && _lru.First is { } oldest) + { + _lru.RemoveFirst(); + if (_entries.Remove(oldest.Value, out Entry? removed)) + _bytes -= removed.Bytes.LongLength; + } + } +} diff --git a/src/AcDream.Content/Pak/PakWriter.cs b/src/AcDream.Content/Pak/PakWriter.cs index 672bbfcd..decb548b 100644 --- a/src/AcDream.Content/Pak/PakWriter.cs +++ b/src/AcDream.Content/Pak/PakWriter.cs @@ -1,7 +1,9 @@ +using System.Buffers; using System; using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; namespace AcDream.Content.Pak; @@ -19,8 +21,16 @@ public sealed class PakWriter : IDisposable { private readonly List _tocEntries = new(); private readonly HashSet _seenKeys = new(); private readonly Dictionary _receiptByKey = new(); + private readonly Dictionary _texturePayloadByKey = new(); private bool _finished; + public int TextureBlobCount { get; private set; } + public int EntryCount => _tocEntries.Count; + public int PhysicalBlobCount { get; private set; } + public long DecodedPayloadBytes { get; private set; } + public long StoredPayloadBytes { get; private set; } + public int CompressedBlobCount { get; private set; } + public PakWriter(string path, PakHeader headerTemplate) { _stream = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None); _headerTemplate = headerTemplate; @@ -49,42 +59,103 @@ public sealed class PakWriter : IDisposable { ReserveKey(key); using var ms = new MemoryStream(); - ObjectMeshDataSerializer.Write(data, ms); - var bytes = ms.ToArray(); - AddReservedBlob(key, bytes); + ObjectMeshDataSerializer.WriteExternalTextures( + data, + ms, + RegisterTexturePayload); + if (!ms.TryGetBuffer(out ArraySegment buffer)) + throw new InvalidOperationException("mesh serializer did not expose its write buffer"); + AddReservedBlob(key, buffer.AsSpan(0, checked((int)ms.Length))); } /// /// Adds an already serialized immutable payload. This is the package seam /// used by typed non-render assets such as flat collision records. /// - public void AddBlob(ulong key, ReadOnlySpan bytes) { + public void AddBlob(ulong key, byte[] bytes) { + ArgumentNullException.ThrowIfNull(bytes); ThrowIfFinished(); ReserveKey(key); AddReservedBlob(key, bytes); } private void AddReservedBlob(ulong key, ReadOnlySpan bytes) { - if ((ulong)bytes.Length > uint.MaxValue) { - throw new ArgumentException( - "one pak payload cannot exceed UInt32.MaxValue bytes", - nameof(bytes)); + bool compressed = PakBlobCodec.TryCompress( + bytes, + out byte[]? rented, + out int storedLength); + try + { + ReadOnlySpan stored = compressed + ? rented.AsSpan(0, storedLength) + : bytes; + long offset = _stream.Position; + _stream.Write(stored); + PadToAlignment(); + + var receipt = new PakTocEntry { + Key = key, + Offset = (ulong)offset, + Length = checked((uint)stored.Length) + | (compressed ? PakTocEntry.CompressionFlag : 0u), + Crc32 = Content.Pak.Crc32.Compute(stored), + }; + _tocEntries.Add(receipt); + _receiptByKey.Add(key, receipt); + PhysicalBlobCount++; + DecodedPayloadBytes = checked(DecodedPayloadBytes + bytes.Length); + StoredPayloadBytes = checked(StoredPayloadBytes + stored.Length); + if (compressed) + CompressedBlobCount++; + } + finally + { + if (rented is not null) + ArrayPool.Shared.Return(rented); + } + } + + private ulong RegisterTexturePayload(TextureKey _, byte[] bytes) + { + Span digest = stackalloc byte[SHA256.HashSizeInBytes]; + SHA256.HashData(bytes, digest); + ulong payloadId = + ((ulong)digest[0] << 48) + | ((ulong)digest[1] << 40) + | ((ulong)digest[2] << 32) + | ((ulong)digest[3] << 24) + | ((ulong)digest[4] << 16) + | ((ulong)digest[5] << 8) + | digest[6]; + ulong key = PakKey.ComposeOpaque( + PakAssetType.TexturePayload, + payloadId); + + if (_texturePayloadByKey.TryGetValue(key, out TextureIdentity existing)) + { + if (existing.Length != bytes.Length + || !CryptographicOperations.FixedTimeEquals( + existing.Sha256, + digest)) + { + throw new InvalidDataException( + $"texture payload identity collision at key 0x{key:X16}"); + } + + return key; } - long offset = _stream.Position; - _stream.Write(bytes); - PadToAlignment(); - - var receipt = new PakTocEntry { - Key = key, - Offset = (ulong)offset, - Length = checked((uint)bytes.Length), - Crc32 = Content.Pak.Crc32.Compute(bytes), - }; - _tocEntries.Add(receipt); - _receiptByKey.Add(key, receipt); + ReserveKey(key); + AddReservedBlob(key, bytes); + _texturePayloadByKey.Add( + key, + new TextureIdentity(bytes.Length, digest.ToArray())); + TextureBlobCount++; + return key; } + private readonly record struct TextureIdentity(int Length, byte[] Sha256); + private void ReserveKey(ulong key) { if (!_seenKeys.Add(key)) { throw new ArgumentException($"duplicate pak key 0x{key:X16}", nameof(key)); diff --git a/src/AcDream.Launcher.Core/Installation/ContentMigrationCatalog.cs b/src/AcDream.Launcher.Core/Installation/ContentMigrationCatalog.cs index 500d5fc9..f31c2085 100644 --- a/src/AcDream.Launcher.Core/Installation/ContentMigrationCatalog.cs +++ b/src/AcDream.Launcher.Core/Installation/ContentMigrationCatalog.cs @@ -45,6 +45,10 @@ public static class ContentMigrationCatalog 4, 5, "solid-colour positive mesh faces must be regenerated"), + [6] = FullRebuild( + 5, + 6, + "the optimized pak v2 texture catalog and compression format require one full rebuild"), }; public static ContentMigrationPlan Resolve(uint fromRecipeVersion, uint targetRecipeVersion) diff --git a/src/AcDream.Launcher.Core/Installation/LauncherContentStateStore.cs b/src/AcDream.Launcher.Core/Installation/LauncherContentStateStore.cs index 6e0e5e84..7776fd52 100644 --- a/src/AcDream.Launcher.Core/Installation/LauncherContentStateStore.cs +++ b/src/AcDream.Launcher.Core/Installation/LauncherContentStateStore.cs @@ -31,7 +31,7 @@ public sealed record LauncherContentState( public sealed class LauncherContentStateStore { private const uint PakMagic = 0x4B504341u; - private const uint PakFormatVersion = 1; + private const uint PakFormatVersion = 2; private const int PakHeaderSize = 64; private static readonly JsonSerializerOptions SerializerOptions = new() diff --git a/src/AcDream.Launcher.Core/Installation/LauncherInstallRecordStore.cs b/src/AcDream.Launcher.Core/Installation/LauncherInstallRecordStore.cs index f96badb7..4d4b3ec6 100644 --- a/src/AcDream.Launcher.Core/Installation/LauncherInstallRecordStore.cs +++ b/src/AcDream.Launcher.Core/Installation/LauncherInstallRecordStore.cs @@ -35,9 +35,9 @@ public sealed record InstallRecordVerification( public sealed class LauncherInstallRecordStore { // Kept in lockstep with AcDream.Content.Pak.PakFormat.CurrentBakeToolVersion - // (#426, 2026-08-23: version 5 extracts untextured/solid-colour positive - // faces that versions <=4 dropped). - public const uint CurrentBakeToolVersion = 5; + // Version 6 is pak format 2: global texture payload deduplication plus + // adaptive independent-blob compression. It is a mandatory full rebuild. + public const uint CurrentBakeToolVersion = 6; private static readonly JsonSerializerOptions SerializerOptions = new() { diff --git a/src/AcDream.Launcher.Core/Installation/LauncherInstaller.cs b/src/AcDream.Launcher.Core/Installation/LauncherInstaller.cs index ea5b9949..f936aaa3 100644 --- a/src/AcDream.Launcher.Core/Installation/LauncherInstaller.cs +++ b/src/AcDream.Launcher.Core/Installation/LauncherInstaller.cs @@ -106,12 +106,15 @@ public interface ILauncherInstaller /// public sealed class LauncherInstaller : ILauncherInstaller { + public const long FullRebuildRequiredFreeBytes = 2L * 1024 * 1024 * 1024; + private readonly string _bakeExecutablePath; private readonly DatDirectoryLocator _datDirectories; private readonly LauncherInstallRecordStore _recordStore; private readonly LauncherContentStateStore _contentStateStore; private readonly IBakeProcessRunner _processRunner; private readonly Func> _computeSha256; + private readonly Func _availableFreeSpace; private readonly SemaphoreSlim _installGate = new(1, 1); private LauncherInstallRecord? _verifiedRecord; @@ -128,7 +131,8 @@ public sealed class LauncherInstaller : ILauncherInstaller LauncherInstallRecordStore? recordStore = null, LauncherContentStateStore? contentStateStore = null, IBakeProcessRunner? processRunner = null, - Func>? computeSha256 = null) + Func>? computeSha256 = null, + Func? availableFreeSpace = null) { ArgumentNullException.ThrowIfNull(paths); ArgumentException.ThrowIfNullOrWhiteSpace(bakeExecutablePath); @@ -145,6 +149,7 @@ public sealed class LauncherInstaller : ILauncherInstaller _contentStateStore = contentStateStore ?? new LauncherContentStateStore(paths, _computeSha256); _processRunner = processRunner ?? new SystemBakeProcessRunner(); + _availableFreeSpace = availableFreeSpace ?? GetAvailableFreeSpace; } public IReadOnlyList DetectDatDirectories() => @@ -302,10 +307,37 @@ public sealed class LauncherInstaller : ILauncherInstaller ?? throw new InvalidOperationException( "The prepared package path has no parent directory.")); + long availableBytes; + try + { + availableBytes = _availableFreeSpace(outputPath); + } + catch (Exception exception) when (exception is IOException + or UnauthorizedAccessException + or ArgumentException) + { + string message = + $"Could not check free space for the world-data rebuild: {exception.Message}"; + Report(progress, LauncherInstallPhase.Failed, message); + throw new LauncherInstallException(message, exception); + } + + if (availableBytes < FullRebuildRequiredFreeBytes) + { + string message = + $"The optimized world-data rebuild needs at least " + + $"{FullRebuildRequiredFreeBytes / (1024d * 1024d * 1024d):N1} GiB free " + + $"beside the active package; only " + + $"{availableBytes / (1024d * 1024d * 1024d):N1} GiB is available."; + Report(progress, LauncherInstallPhase.Failed, message); + throw new LauncherInstallException(message); + } + Report( progress, LauncherInstallPhase.PreparingOutput, - "Preparing a replacement beside the active package..."); + "Preparing a one-time optimized world-data rebuild beside the active package. " + + "This can take several minutes; progress will update below..."); LauncherInstallRecordStore.TryDelete(bakeOutputPath); LauncherInstallRecordStore.TryDelete(backupPath); bool previousPreserved = false; @@ -1156,6 +1188,16 @@ public sealed class LauncherInstaller : ILauncherInstaller internal static string GetFullRebuildCandidatePath(string outputPath) => outputPath + ".candidate"; + private static long GetAvailableFreeSpace(string path) + { + string fullPath = Path.GetFullPath(path); + string root = Path.GetPathRoot(fullPath) + ?? throw new ArgumentException( + $"Path '{fullPath}' has no filesystem root.", + nameof(path)); + return new DriveInfo(root).AvailableFreeSpace; + } + private static string BuildChildFailure( int exitCode, string? jsonError, diff --git a/src/AcDream.Launcher/ViewModels/FirstRunInstallerViewModel.cs b/src/AcDream.Launcher/ViewModels/FirstRunInstallerViewModel.cs index 0391b225..f5bff4c8 100644 --- a/src/AcDream.Launcher/ViewModels/FirstRunInstallerViewModel.cs +++ b/src/AcDream.Launcher/ViewModels/FirstRunInstallerViewModel.cs @@ -466,10 +466,9 @@ public sealed class FirstRunInstallerViewModel : ObservableObject, IDisposable string estimate = migration.Kind == ContentWorkKind.Overlay ? $"Affected filters: {migration.EffectiveDatIds.Count:N0} DAT id(s), " + $"{migration.EffectiveLandblocks.Count:N0} landblock(s)." - : _contentUpdateBase is { PreparedAssetSize: > 0 } record - ? $"Free-space guidance: allow about " - + $"{Math.Ceiling(record.PreparedAssetSize * 1.1 / (1024d * 1024d * 1024d)):N0} GiB." - : "Free-space guidance: allow room for one complete replacement pak."; + : $"Free-space guidance: allow at least " + + $"{LauncherInstaller.FullRebuildRequiredFreeBytes / (1024d * 1024d * 1024d):N0} GiB " + + "while the optimized package is built beside the active one."; return $"This client needs recipe {migration.TargetRecipeVersion}: " + $"{migration.Reason}. acdream will build {work} from your installed " + "Asheron's Call DAT files. The existing package stays in place " diff --git a/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs b/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs index cb747abe..df7ba69b 100644 --- a/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs +++ b/tests/AcDream.Bake.Tests/BakeDeterminismTests.cs @@ -153,7 +153,7 @@ public sealed class BakeDeterminismTests : IDisposable { Assert.Equal(1, reportA.UniqueCellStructureCollisions); Assert.Equal(7, reportA.CellStructureCollisionAliases); Assert.Equal(8, reportA.EnvCellTopologyKeys); - Assert.Equal(10, reportA.PhysicalBlobs); + Assert.Equal(10 + reportA.TexturePayloadKeys, reportA.PhysicalBlobs); Assert.Equal(reportA.TotalKeys, reportB.TotalKeys); Assert.True(File.ReadAllBytes(pathA).AsSpan().SequenceEqual(File.ReadAllBytes(pathB))); diff --git a/tests/AcDream.Content.Tests/InstalledPreparedCollisionCatalogTests.cs b/tests/AcDream.Content.Tests/InstalledPreparedCollisionCatalogTests.cs index a3008275..186ecde2 100644 --- a/tests/AcDream.Content.Tests/InstalledPreparedCollisionCatalogTests.cs +++ b/tests/AcDream.Content.Tests/InstalledPreparedCollisionCatalogTests.cs @@ -14,7 +14,7 @@ public sealed class InstalledPreparedCollisionCatalogTests if (datDir is null) Assert.Fail("Lane=PreparedPackage requires installed retail DATs and a validated acdream.pak; see docs/release-gate.md."); - string packagePath = Path.Combine(datDir, "acdream.pak"); + string packagePath = ResolvePackagePath(datDir); if (!File.Exists(packagePath)) Assert.Fail("Lane=PreparedPackage requires installed retail DATs and a validated acdream.pak; see docs/release-gate.md."); @@ -36,7 +36,17 @@ public sealed class InstalledPreparedCollisionCatalogTests PreparedAssetReadStatus.Loaded, source.ReadEnvCellTopology(0xA9B4_013Fu).Status); Assert.Equal(4, source.CollisionStats.Loaded); - Assert.True(source.MappedVirtualBytes > 1L << 30); + Assert.Equal(new FileInfo(packagePath).Length, source.MappedVirtualBytes); + Assert.InRange(source.MappedVirtualBytes, 1L, 5L * 1024 * 1024 * 1024); + } + + private static string ResolvePackagePath(string datDir) + { + string? configured = + Environment.GetEnvironmentVariable("ACDREAM_PAK_PATH"); + return !string.IsNullOrWhiteSpace(configured) + ? Path.GetFullPath(configured) + : Path.Combine(datDir, "acdream.pak"); } private static string? ResolveDatDir() diff --git a/tests/AcDream.Content.Tests/MeshExtractorSolidFaceExtractionTests.cs b/tests/AcDream.Content.Tests/MeshExtractorSolidFaceExtractionTests.cs index 7763e7c6..00121fa1 100644 --- a/tests/AcDream.Content.Tests/MeshExtractorSolidFaceExtractionTests.cs +++ b/tests/AcDream.Content.Tests/MeshExtractorSolidFaceExtractionTests.cs @@ -4,6 +4,7 @@ using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Numerics; using AcDream.Content; +using Chorizite.Core.Render.Enums; using DatReaderWriter; using DatReaderWriter.DBObjs; using DatReaderWriter.Enums; @@ -112,6 +113,100 @@ public sealed class MeshExtractorSolidFaceExtractionTests Assert.False(batch.Key.IsSolid); } + [Theory] + [InlineData(PixelFormat.PFID_DXT1, TextureFormat.DXT1, 8)] + [InlineData(PixelFormat.PFID_DXT3, TextureFormat.DXT3, 16)] + [InlineData(PixelFormat.PFID_DXT5, TextureFormat.DXT5, 16)] + public void PrepareMeshData_UneditedDxtSurface_PreservesNativeBlocks( + PixelFormat sourceFormat, + TextureFormat expectedFormat, + int sourceBytes) + { + var dats = new FakeMeshExtractorDats(); + byte[] blocks = Enumerable.Range(0, sourceBytes).Select(i => (byte)i).ToArray(); + RegisterTexturedQuad(dats, SurfaceType.Base1Image, sourceFormat, blocks); + + var extractor = new MeshExtractor(dats, NullLogger.Instance, sideStagedSink: null); + ObjectMeshData mesh = Assert.IsType( + extractor.PrepareMeshData(GfxObjId, isSetup: false)); + + KeyValuePair<(int Width, int Height, TextureFormat Format), List> group = + Assert.Single(mesh.TextureBatches); + Assert.Equal(expectedFormat, group.Key.Format); + TextureBatchData batch = Assert.Single(group.Value); + Assert.Same(blocks, batch.TextureData); + Assert.Null(batch.UploadPixelFormat); + Assert.Null(batch.UploadPixelType); + } + + [Fact] + public void PrepareMeshData_DxtClipMap_DecodesForSurfaceLocalAlphaEdit() + { + var dats = new FakeMeshExtractorDats(); + RegisterTexturedQuad( + dats, + SurfaceType.Base1Image | SurfaceType.Base1ClipMap, + PixelFormat.PFID_DXT1, + new byte[8]); + + var extractor = new MeshExtractor(dats, NullLogger.Instance, sideStagedSink: null); + ObjectMeshData mesh = Assert.IsType( + extractor.PrepareMeshData(GfxObjId, isSetup: false)); + + KeyValuePair<(int Width, int Height, TextureFormat Format), List> group = + Assert.Single(mesh.TextureBatches); + Assert.Equal(TextureFormat.RGBA8, group.Key.Format); + Assert.Equal(4 * 4 * 4, Assert.Single(group.Value).TextureData.Length); + } + + [Fact] + public void PrepareMeshData_TranslucentDxt_DecodesForAlphaScale() + { + var dats = new FakeMeshExtractorDats(); + RegisterTexturedQuad( + dats, + SurfaceType.Base1Image, + PixelFormat.PFID_DXT1, + new byte[8], + translucency: 0.25f); + + var extractor = new MeshExtractor(dats, NullLogger.Instance, sideStagedSink: null); + ObjectMeshData mesh = Assert.IsType( + extractor.PrepareMeshData(GfxObjId, isSetup: false)); + + KeyValuePair<(int Width, int Height, TextureFormat Format), List> group = + Assert.Single(mesh.TextureBatches); + Assert.Equal(TextureFormat.RGBA8, group.Key.Format); + Assert.Equal(4 * 4 * 4, Assert.Single(group.Value).TextureData.Length); + } + + private static void RegisterTexturedQuad( + FakeMeshExtractorDats dats, + SurfaceType surfaceType, + PixelFormat pixelFormat, + byte[] sourceData, + float translucency = 0.0f) + { + dats.RegisterRootGfxObj(GfxObjId, BuildQuadGfxObj(TexturedSurfaceId, noPos: false)); + dats.Register(TexturedSurfaceId, new Surface + { + Type = surfaceType, + OrigTextureId = SurfaceTextureId, + Translucency = translucency, + }); + dats.Register(SurfaceTextureId, new SurfaceTexture + { + Textures = new List> { RenderSurfaceId }, + }); + dats.Register(RenderSurfaceId, new RenderSurface + { + Width = 4, + Height = 4, + Format = pixelFormat, + SourceData = sourceData, + }); + } + /// One quad (4 verts), single polygon, PosSurface referencing . private static GfxObj BuildQuadGfxObj(uint surfaceId, bool noPos) { diff --git a/tests/AcDream.Content.Tests/PakEquivalenceTests.cs b/tests/AcDream.Content.Tests/PakEquivalenceTests.cs index b0245220..a044c331 100644 --- a/tests/AcDream.Content.Tests/PakEquivalenceTests.cs +++ b/tests/AcDream.Content.Tests/PakEquivalenceTests.cs @@ -88,7 +88,7 @@ public sealed class PakEquivalenceTests { var pakPath = Path.Combine(Path.GetTempPath(), $"acdream-equivtest-{System.Guid.NewGuid():N}.pak"); try { var header = new PakHeader { - FormatVersion = 1, + FormatVersion = PakFormat.CurrentFormatVersion, PortalIteration = (uint)dats.Portal.Iteration!.CurrentIteration, CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration, HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration, diff --git a/tests/AcDream.Content.Tests/PakRoundTripTests.cs b/tests/AcDream.Content.Tests/PakRoundTripTests.cs index 14986614..f76bfb49 100644 --- a/tests/AcDream.Content.Tests/PakRoundTripTests.cs +++ b/tests/AcDream.Content.Tests/PakRoundTripTests.cs @@ -50,7 +50,7 @@ public class PakRoundTripTests : IDisposable { private static PakHeader WritePak(string path, (PakAssetType Type, uint FileId, ObjectMeshData Data)[] blobs) { var header = new PakHeader { - FormatVersion = 1, + FormatVersion = PakFormat.CurrentFormatVersion, PortalIteration = 10, CellIteration = 20, HighResIteration = 30, @@ -231,7 +231,8 @@ public class PakRoundTripTests : IDisposable { [Theory] [InlineData(0u)] - [InlineData(2u)] + [InlineData(1u)] + [InlineData(3u)] public void Reader_RejectsWrongFormatVersion(uint wrongVersion) { var path = NewTempPakPath(); WritePak(path, MakeBlobSet(2)); @@ -339,7 +340,7 @@ public class PakRoundTripTests : IDisposable { // Recompute CRC over the tampered blob region. fs.Position = (long)entry.Offset; - var blobBytes = new byte[entry.Length]; + var blobBytes = new byte[entry.StoredLength]; fs.ReadExactly(blobBytes); uint newCrc = Crc32.Compute(blobBytes); diff --git a/tests/AcDream.Content.Tests/PakV2Tests.cs b/tests/AcDream.Content.Tests/PakV2Tests.cs new file mode 100644 index 00000000..50b9d158 --- /dev/null +++ b/tests/AcDream.Content.Tests/PakV2Tests.cs @@ -0,0 +1,292 @@ +using System.Buffers.Binary; +using System.Security.Cryptography; +using AcDream.Content.Pak; +using Chorizite.Core.Render.Enums; +using DatReaderWriter.Enums; +using RetailCullMode = DatReaderWriter.Enums.CullMode; + +namespace AcDream.Content.Tests; + +public sealed class PakV2Tests : IDisposable +{ + private readonly List _paths = []; + + [Fact] + public void BlobCodec_UsesRawFallbackAndCompressedRoundTripsExactly() + { + byte[] small = [1, 2, 3, 4]; + PakBlobCodec.Encoded raw = PakBlobCodec.Encode(small); + Assert.False(raw.Compressed); + Assert.Equal(small, raw.Bytes); + + byte[] repetitive = new byte[64 * 1024]; + for (int i = 0; i < repetitive.Length; i++) + repetitive[i] = (byte)(i & 7); + + PakBlobCodec.Encoded compressed = PakBlobCodec.Encode(repetitive); + Assert.True(compressed.Compressed); + Assert.True(compressed.Bytes.Length < repetitive.Length / 4); + Assert.Equal(repetitive, PakBlobCodec.Decode(compressed.Bytes)); + } + + [Theory] + [InlineData(new byte[] { })] + [InlineData(new byte[] { 1, 2, 3 })] + [InlineData(new byte[] { 4, 0, 0, 0, 255 })] + public void BlobCodec_RejectsMalformedCompressedPayload(byte[] stored) => + Assert.Throws(() => PakBlobCodec.Decode(stored)); + + [Fact] + public void BlobCodec_RejectsAllocationBombBeforeAllocating() + { + byte[] stored = new byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian( + stored, + PakBlobCodec.MaximumDecodedBytes + 1u); + + Assert.Throws(() => PakBlobCodec.Decode(stored)); + } + + [Fact] + public void Writer_DeduplicatesTextureBytesGloballyAndReaderSharesArray() + { + string path = NewPath(); + byte[] texture = new byte[16 * 1024]; + for (int i = 0; i < texture.Length; i++) + texture[i] = (byte)(i & 15); + + ObjectMeshData first = TexturedMesh(1, 0x0800_0001u, texture); + ObjectMeshData second = TexturedMesh(2, 0x0800_0002u, texture.ToArray()); + using (var writer = NewWriter(path)) + { + writer.AddBlob(PakKey.Compose(PakAssetType.GfxObjMesh, 1), first); + writer.AddBlob(PakKey.Compose(PakAssetType.GfxObjMesh, 2), second); + Assert.Equal(1, writer.TextureBlobCount); + Assert.Equal(3, writer.EntryCount); + writer.Finish(); + } + + using var reader = new PakReader(path); + Assert.Equal(1, reader.CountEntries(PakAssetType.TexturePayload)); + Assert.True(reader.TryReadObjectMeshData( + PakKey.Compose(PakAssetType.GfxObjMesh, 1), + out ObjectMeshData? firstRead)); + Assert.True(reader.TryReadObjectMeshData( + PakKey.Compose(PakAssetType.GfxObjMesh, 2), + out ObjectMeshData? secondRead)); + ObjectMeshDataEquality.AssertEqual(first, firstRead); + ObjectMeshDataEquality.AssertEqual(second, secondRead); + Assert.Same( + firstRead!.TextureBatches.Single().Value.Single().TextureData, + secondRead!.TextureBatches.Single().Value.Single().TextureData); + } + + [Fact] + public void Writer_IsByteDeterministicWithExternalTexturesAndCompression() + { + string firstPath = NewPath(); + string secondPath = NewPath(); + byte[] texture = Enumerable.Repeat((byte)0x5A, 32 * 1024).ToArray(); + + WriteDeterministic(firstPath, texture); + WriteDeterministic(secondPath, texture); + + Assert.Equal(File.ReadAllBytes(firstPath), File.ReadAllBytes(secondPath)); + } + + [Fact] + public void CorruptTextureDemotesOnlyReferencingMesh() + { + string path = NewPath(); + byte[] badTexture = Enumerable.Repeat((byte)0x11, 16 * 1024).ToArray(); + byte[] goodTexture = Enumerable.Repeat((byte)0x22, 16 * 1024).ToArray(); + ulong badMeshKey = PakKey.Compose(PakAssetType.GfxObjMesh, 1); + ulong goodMeshKey = PakKey.Compose(PakAssetType.GfxObjMesh, 2); + ulong badTextureKey = TexturePayloadKey(badTexture); + using (var writer = NewWriter(path)) + { + writer.AddBlob(badMeshKey, TexturedMesh(1, 1, badTexture)); + writer.AddBlob(goodMeshKey, TexturedMesh(2, 2, goodTexture)); + writer.Finish(); + } + + PakTocEntry textureEntry; + using (var reader = new PakReader(path)) + textureEntry = reader.GetTocEntryForTest(badTextureKey); + + using (var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite)) + { + stream.Position = (long)textureEntry.Offset + textureEntry.StoredLength / 2; + int value = stream.ReadByte(); + stream.Position--; + stream.WriteByte((byte)(value ^ 0xFF)); + } + + using var corruptReader = new PakReader(path); + Assert.Equal( + PakObjectReadStatus.Corrupt, + corruptReader.ReadObjectMeshData(badMeshKey, out _)); + Assert.Equal(PakEntryState.Corrupt, corruptReader.ProbeEntry(badTextureKey)); + Assert.Equal( + PakObjectReadStatus.Loaded, + corruptReader.ReadObjectMeshData(goodMeshKey, out ObjectMeshData? good)); + Assert.Equal(goodTexture, good!.TextureBatches.Single().Value.Single().TextureData); + } + + [Fact] + public void Reader_DemotesInvalidCompressedStreamBehindValidCrc() + { + string path = NewPath(); + ulong key = PakKey.Compose(PakAssetType.GfxObjCollision, 7); + using (var writer = NewWriter(path)) + { + writer.AddBlob(key, new byte[32 * 1024]); + writer.Finish(); + } + + PakTocEntry entry; + long tocEntryPosition; + using (var reader = new PakReader(path)) + { + entry = reader.GetTocEntryForTest(key); + Assert.True(entry.IsCompressed); + tocEntryPosition = FindTocEntryPosition(path, key); + } + + byte[] invalid = new byte[checked((int)entry.StoredLength)]; + BinaryPrimitives.WriteUInt32LittleEndian(invalid, 32 * 1024); + invalid.AsSpan(sizeof(uint)).Fill(0xFF); + using (var stream = new FileStream(path, FileMode.Open, FileAccess.ReadWrite)) + { + stream.Position = (long)entry.Offset; + stream.Write(invalid); + stream.Position = tocEntryPosition + 20; + Span crc = stackalloc byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(crc, Crc32.Compute(invalid)); + stream.Write(crc); + } + + using var corruptReader = new PakReader(path); + Assert.Equal( + PakObjectReadStatus.Corrupt, + corruptReader.ReadBlobBytes(key, out byte[]? bytes)); + Assert.Null(bytes); + Assert.Equal(PakEntryState.Corrupt, corruptReader.ProbeEntry(key)); + } + + [Fact] + public void TextureCache_IsReferenceSharingAndStrictlyBounded() + { + var cache = new PakTexturePayloadCache(maximumBytes: 6, maximumEntries: 2); + byte[] first = [1, 2, 3, 4]; + byte[] second = [5, 6, 7, 8]; + + Assert.Same(first, cache.AddOrGet(1, first)); + Assert.Same(first, cache.AddOrGet(1, first.ToArray())); + Assert.Same(second, cache.AddOrGet(2, second)); + Assert.False(cache.TryGet(1, out _)); + Assert.True(cache.TryGet(2, out byte[] cachedSecond)); + Assert.Same(second, cachedSecond); + Assert.Equal(1, cache.Count); + Assert.Equal(4, cache.Bytes); + + byte[] tooLarge = new byte[7]; + Assert.Same(tooLarge, cache.AddOrGet(3, tooLarge)); + Assert.False(cache.TryGet(3, out _)); + Assert.Equal(4, cache.Bytes); + } + + private static ObjectMeshData TexturedMesh( + uint objectId, + uint surfaceId, + byte[] texture) + { + var mesh = new ObjectMeshData { ObjectId = objectId }; + mesh.TextureBatches[(64, 64, TextureFormat.RGBA8)] = + [ + new TextureBatchData + { + Key = new TextureKey + { + SurfaceId = surfaceId, + PaletteId = 1, + Stippling = StipplingType.Positive, + }, + TextureData = texture, + Indices = [0, 1, 2], + CullMode = RetailCullMode.Clockwise, + }, + ]; + return mesh; + } + + private static PakWriter NewWriter(string path) => + new(path, new PakHeader + { + PortalIteration = 1, + CellIteration = 2, + HighResIteration = 3, + LanguageIteration = 4, + }); + + private static void WriteDeterministic(string path, byte[] texture) + { + using var writer = NewWriter(path); + writer.AddBlob( + PakKey.Compose(PakAssetType.GfxObjMesh, 2), + TexturedMesh(2, 12, texture.ToArray())); + writer.AddBlob( + PakKey.Compose(PakAssetType.GfxObjMesh, 1), + TexturedMesh(1, 11, texture.ToArray())); + writer.Finish(); + } + + private static ulong TexturePayloadKey(byte[] bytes) + { + byte[] digest = SHA256.HashData(bytes); + ulong payloadId = + ((ulong)digest[0] << 48) + | ((ulong)digest[1] << 40) + | ((ulong)digest[2] << 32) + | ((ulong)digest[3] << 24) + | ((ulong)digest[4] << 16) + | ((ulong)digest[5] << 8) + | digest[6]; + return PakKey.ComposeOpaque(PakAssetType.TexturePayload, payloadId); + } + + private string NewPath() + { + string path = Path.Combine( + Path.GetTempPath(), + $"acdream-pak-v2-{Guid.NewGuid():N}.pak"); + _paths.Add(path); + return path; + } + + private static long FindTocEntryPosition(string path, ulong key) + { + using var stream = new FileStream(path, FileMode.Open, FileAccess.Read); + PakHeader header = PakHeader.ReadFrom(stream); + var bytes = new byte[PakTocEntry.Size]; + for (uint i = 0; i < header.TocCount; i++) + { + long position = checked((long)header.TocOffset + i * PakTocEntry.Size); + stream.Position = position; + stream.ReadExactly(bytes); + if (PakTocEntry.ReadFrom(bytes).Key == key) + return position; + } + + throw new KeyNotFoundException($"pak key 0x{key:X16} not found"); + } + + public void Dispose() + { + foreach (string path in _paths) + { + try { File.Delete(path); } + catch (IOException) { } + } + } +} diff --git a/tests/AcDream.Launcher.Core.Tests/Installation/ContentMigrationCatalogTests.cs b/tests/AcDream.Launcher.Core.Tests/Installation/ContentMigrationCatalogTests.cs new file mode 100644 index 00000000..f73b3fac --- /dev/null +++ b/tests/AcDream.Launcher.Core.Tests/Installation/ContentMigrationCatalogTests.cs @@ -0,0 +1,29 @@ +using AcDream.Launcher.Core.Installation; + +namespace AcDream.Launcher.Core.Tests.Installation; + +public sealed class ContentMigrationCatalogTests +{ + [Fact] + public void RecipeFiveToSixRequiresOneExplicitFullRebuild() + { + ContentMigrationPlan plan = ContentMigrationCatalog.Resolve(5, 6); + + Assert.Equal(ContentWorkKind.FullRebuild, plan.Kind); + Assert.Equal(5u, plan.FromRecipeVersion); + Assert.Equal(6u, plan.TargetRecipeVersion); + Assert.Contains("pak v2", plan.Reason, StringComparison.OrdinalIgnoreCase); + Assert.Empty(plan.EffectiveDatIds); + Assert.Empty(plan.EffectiveLandblocks); + } + + [Fact] + public void AnyOlderRecipeToSixCollapsesToOneFullRebuild() + { + ContentMigrationPlan plan = ContentMigrationCatalog.Resolve(1, 6); + + Assert.Equal(ContentWorkKind.FullRebuild, plan.Kind); + Assert.Equal(6u, plan.TargetRecipeVersion); + Assert.Contains("pak v2", plan.Reason, StringComparison.OrdinalIgnoreCase); + } +} diff --git a/tests/AcDream.Launcher.Core.Tests/Installation/LauncherContentStateStoreTests.cs b/tests/AcDream.Launcher.Core.Tests/Installation/LauncherContentStateStoreTests.cs index 6e3841a1..9a1b5a08 100644 --- a/tests/AcDream.Launcher.Core.Tests/Installation/LauncherContentStateStoreTests.cs +++ b/tests/AcDream.Launcher.Core.Tests/Installation/LauncherContentStateStoreTests.cs @@ -133,7 +133,7 @@ public sealed class LauncherContentStateStoreTests : IDisposable Directory.CreateDirectory(Path.GetDirectoryName(path)!); byte[] bytes = new byte[64]; BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(0, 4), 0x4B504341u); - BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(4, 4), 1); + BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(4, 4), 2); BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(8, 4), 100); BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(12, 4), 200); BinaryPrimitives.WriteUInt32LittleEndian(bytes.AsSpan(16, 4), 300); diff --git a/tests/AcDream.Launcher.Core.Tests/Installation/LauncherInstallerTests.cs b/tests/AcDream.Launcher.Core.Tests/Installation/LauncherInstallerTests.cs index ddd842e2..dcd326f9 100644 --- a/tests/AcDream.Launcher.Core.Tests/Installation/LauncherInstallerTests.cs +++ b/tests/AcDream.Launcher.Core.Tests/Installation/LauncherInstallerTests.cs @@ -147,6 +147,33 @@ public sealed class LauncherInstallerTests : IDisposable StringComparison.Ordinal); } + [Fact] + public async Task InsufficientFreeSpaceStopsBeforeBakeAndReportsExactRequirement() + { + bool childStarted = false; + var runner = new FakeBakeProcessRunner((_, _, _) => + { + childStarted = true; + return Task.FromResult(new BakeProcessResult(0, string.Empty)); + }); + var installer = new LauncherInstaller( + _paths, + _bakeExecutable, + processRunner: runner, + availableFreeSpace: _ => + LauncherInstaller.FullRebuildRequiredFreeBytes - 1); + + LauncherInstallException error = await Assert.ThrowsAsync( + () => installer.InstallAsync(_dats, threads: 2)); + + Assert.False(childStarted); + Assert.Contains("2.0 GiB", error.Message, StringComparison.Ordinal); + Assert.Contains("active package", error.Message, StringComparison.Ordinal); + Assert.False(File.Exists( + LauncherInstaller.GetFullRebuildCandidatePath( + Path.Combine(_paths.DataDirectory, "pak", "acdream.pak")))); + } + [Fact] public async Task FullContentMigrationPersistsClientGateAcrossLauncherRestart() { diff --git a/tests/AcDream.Launcher.Core.Tests/Installation/LauncherOverlayInstallerTests.cs b/tests/AcDream.Launcher.Core.Tests/Installation/LauncherOverlayInstallerTests.cs index c0b0e4da..39162060 100644 --- a/tests/AcDream.Launcher.Core.Tests/Installation/LauncherOverlayInstallerTests.cs +++ b/tests/AcDream.Launcher.Core.Tests/Installation/LauncherOverlayInstallerTests.cs @@ -58,8 +58,8 @@ public sealed class LauncherOverlayInstallerTests : IDisposable request.OutputPath, LauncherInstallRecordStore.CurrentBakeToolVersion); long bytes = new FileInfo(request.OutputPath).Length; - output($"{{\"v\":1,\"e\":\"started\",\"bakeToolVersion\":5}}\n"); - output($"{{\"v\":1,\"e\":\"completed\",\"bakeToolVersion\":5," + output($"{{\"v\":1,\"e\":\"started\",\"bakeToolVersion\":{LauncherInstallRecordStore.CurrentBakeToolVersion}}}\n"); + output($"{{\"v\":1,\"e\":\"completed\",\"bakeToolVersion\":{LauncherInstallRecordStore.CurrentBakeToolVersion}," + $"\"outputBytes\":{bytes},\"failures\":0}}\n"); await Task.Yield(); return new BakeProcessResult(0, string.Empty); @@ -70,8 +70,8 @@ public sealed class LauncherOverlayInstallerTests : IDisposable recordStore: recordStore, processRunner: runner); var migration = new ContentMigrationPlan( - 4, - 5, + LauncherInstallRecordStore.CurrentBakeToolVersion - 1, + LauncherInstallRecordStore.CurrentBakeToolVersion, ContentWorkKind.Overlay, "bounded fixture update", [0x01001234u, 0x02005678u], @@ -93,7 +93,9 @@ public sealed class LauncherOverlayInstallerTests : IDisposable observed.Arguments.SkipWhile(value => value != "--landblocks").Take(2)); Assert.NotNull(result.Record.PreparedAssetOverlayPath); Assert.True(File.Exists(result.Record.PreparedAssetOverlayPath)); - Assert.Equal(5u, result.Record.ResolvedBakeToolVersion); + Assert.Equal( + LauncherInstallRecordStore.CurrentBakeToolVersion, + result.Record.ResolvedBakeToolVersion); Assert.True(result.Record.RequiresClientCompatibilityConfirmation); Assert.False(File.Exists( new LauncherContentStateStore(_paths).OverlayCandidatePath)); @@ -145,7 +147,9 @@ public sealed class LauncherOverlayInstallerTests : IDisposable TaskCreationOptions.RunContinuationsAsynchronously); var runner = new FakeRunner(async (request, _, cancellationToken) => { - LauncherContentStateStoreTests.WritePakHeader(request.OutputPath, 5); + LauncherContentStateStoreTests.WritePakHeader( + request.OutputPath, + LauncherInstallRecordStore.CurrentBakeToolVersion); entered.SetResult(); await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); return new BakeProcessResult(0, string.Empty); @@ -160,8 +164,8 @@ public sealed class LauncherOverlayInstallerTests : IDisposable _dats, 2, new ContentMigrationPlan( - 4, - 5, + LauncherInstallRecordStore.CurrentBakeToolVersion - 1, + LauncherInstallRecordStore.CurrentBakeToolVersion, ContentWorkKind.Overlay, "bounded fixture update", [0x01001234u]), @@ -189,11 +193,11 @@ public sealed class LauncherOverlayInstallerTests : IDisposable { LauncherContentStateStoreTests.WritePakHeader( recordStore.PreparedAssetPath, - recipe: 4); + recipe: LauncherInstallRecordStore.CurrentBakeToolVersion - 1); LauncherInstallRecord baseRecord = await LauncherContentStateStoreTests.RecordAsync( recordStore.PreparedAssetPath, - recipe: 4, + recipe: LauncherInstallRecordStore.CurrentBakeToolVersion - 1, datDirectory: Path.GetFullPath(_dats)); Directory.CreateDirectory(_paths.DataDirectory); await File.WriteAllTextAsync( diff --git a/tests/AcDream.Launcher.Tests/LauncherWindowViewModelTests.cs b/tests/AcDream.Launcher.Tests/LauncherWindowViewModelTests.cs index fa61520b..ad90796f 100644 --- a/tests/AcDream.Launcher.Tests/LauncherWindowViewModelTests.cs +++ b/tests/AcDream.Launcher.Tests/LauncherWindowViewModelTests.cs @@ -142,6 +142,7 @@ public sealed class LauncherWindowViewModelTests Assert.Equal("World data update required", viewModel.InstallationBannerTitle); Assert.Contains("complete replacement pak", viewModel.FirstRunWizardShell.Body); Assert.Contains("existing package stays", viewModel.FirstRunWizardShell.Body); + Assert.Contains("2 GiB", viewModel.FirstRunWizardShell.Body); Assert.Equal("Rebuild world data", viewModel.FirstRunWizardShell.StartActionText); Assert.Null(installer.InstallRequest); diff --git a/tools/run-connected-r6-soak.ps1 b/tools/run-connected-r6-soak.ps1 index 3bfdbee7..a9d4a90c 100644 --- a/tools/run-connected-r6-soak.ps1 +++ b/tools/run-connected-r6-soak.ps1 @@ -3,6 +3,7 @@ param( [string]$Repository = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path, [string]$Account = $env:ACDREAM_TEST_USER, [string]$Password = $env:ACDREAM_TEST_PASS, + [string]$PreparedAssetPath, [switch]$SkipBuild, [switch]$Uncapped, [switch]$DenseTown, @@ -748,6 +749,13 @@ $videoControllers = @(Get-CimInstance Win32_VideoController -ErrorAction Silentl }) $env:ACDREAM_DAT_DIR = "$env:USERPROFILE\Documents\Asheron's Call" +if (-not [string]::IsNullOrWhiteSpace($PreparedAssetPath)) { + $resolvedPreparedAssetPath = [IO.Path]::GetFullPath($PreparedAssetPath) + if (-not (Test-Path -LiteralPath $resolvedPreparedAssetPath -PathType Leaf)) { + throw "prepared asset package not found: $resolvedPreparedAssetPath" + } + $env:ACDREAM_PAK_PATH = $resolvedPreparedAssetPath +} $env:ACDREAM_LIVE = '1' $env:ACDREAM_TEST_HOST = '127.0.0.1' $env:ACDREAM_TEST_PORT = '9000' diff --git a/tools/run-connected-world-lifecycle-gate.ps1 b/tools/run-connected-world-lifecycle-gate.ps1 index 704a244d..6f3076f8 100644 --- a/tools/run-connected-world-lifecycle-gate.ps1 +++ b/tools/run-connected-world-lifecycle-gate.ps1 @@ -3,6 +3,7 @@ param( [string]$Repository = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path, [string]$Account = $env:ACDREAM_TEST_USER, [string]$Password = $env:ACDREAM_TEST_PASS, + [string]$PreparedAssetPath, [string]$AceLogPath = 'C:\ACE\Server\ACE_Log.txt', [switch]$SkipBuild, [int]$SessionTimeoutSeconds = 420, @@ -559,6 +560,13 @@ $renderPackGate = New-ConnectedRenderPackGateState ` -Root $root ` -Preset $RenderPackPreset ` -SettingOverrides $RenderPackSettingOverrides +if (-not [string]::IsNullOrWhiteSpace($PreparedAssetPath)) { + $resolvedPreparedAssetPath = [IO.Path]::GetFullPath($PreparedAssetPath) + if (-not (Test-Path -LiteralPath $resolvedPreparedAssetPath -PathType Leaf)) { + throw "prepared asset package not found: $resolvedPreparedAssetPath" + } + $env:ACDREAM_PAK_PATH = $resolvedPreparedAssetPath +} $sessionConfigPath = New-ConnectedGraphicalSessionConfig ` -State $renderPackGate ` -Account $Account