diff --git a/src/AcDream.Content/Pak/Crc32.cs b/src/AcDream.Content/Pak/Crc32.cs new file mode 100644 index 00000000..e2afda95 --- /dev/null +++ b/src/AcDream.Content/Pak/Crc32.cs @@ -0,0 +1,36 @@ +using System; + +namespace AcDream.Content.Pak; + +/// +/// Standard CRC-32 (IEEE 802.3 polynomial 0xEDB88320, the same variant used +/// by zip/gzip/PNG). Implemented directly rather than pulling in +/// System.IO.Hashing to keep AcDream.Content's dependency surface minimal +/// (mirrors the "no GL binaries, keep it lean" intent of the project — see +/// AcDream.Content.csproj's package comments). Used as the pak TOC's +/// per-blob corruption tripwire. +/// +public static class Crc32 { + private static readonly uint[] Table = BuildTable(); + + private static uint[] BuildTable() { + const uint polynomial = 0xEDB88320u; + var table = new uint[256]; + for (uint i = 0; i < 256; i++) { + uint c = i; + for (int k = 0; k < 8; k++) { + c = (c & 1) != 0 ? polynomial ^ (c >> 1) : c >> 1; + } + table[i] = c; + } + return table; + } + + public static uint Compute(ReadOnlySpan data) { + uint crc = 0xFFFFFFFFu; + foreach (var b in data) { + crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8); + } + return crc ^ 0xFFFFFFFFu; + } +} diff --git a/src/AcDream.Content/Pak/PakReader.cs b/src/AcDream.Content/Pak/PakReader.cs new file mode 100644 index 00000000..02b87183 --- /dev/null +++ b/src/AcDream.Content/Pak/PakReader.cs @@ -0,0 +1,139 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.IO.MemoryMappedFiles; + +namespace AcDream.Content.Pak; + +/// +/// Zero-copy mmap reader for an acdream pak file. One +/// for the whole file's lifetime; TOC is loaded once at construction into a +/// key-sorted array for O(log n) binary-search lookup. Blob CRCs are +/// verified LAZILY on first access of each blob (not eagerly at open) and +/// cached so a corrupt blob is only logged once; a corrupted blob is +/// thereafter always treated as "missing" rather than returning garbage. +/// +/// No locks anywhere: the underlying mapped memory is immutable for the +/// lifetime of this reader (acdream never mutates a pak file while it's +/// open for reading — MP1c's cutover writes a NEW pak and swaps it in), so +/// concurrent reads from multiple threads are safe by construction. +/// +public sealed class PakReader : IDisposable { + private readonly MemoryMappedFile _mmf; + private readonly MemoryMappedViewAccessor _accessor; + private readonly long _fileLength; + private readonly PakTocEntry[] _toc; // sorted ascending by Key + + /// -1 = not yet checked, 0 = crc mismatch (corrupt), 1 = crc ok. + private readonly ConcurrentDictionary _crcVerifiedByTocIndex = new(); + private readonly ConcurrentDictionary _loggedCorruption = new(); + + public PakHeader Header { get; } + + public PakReader(string path) { + _fileLength = new FileInfo(path).Length; + if (_fileLength < PakHeader.Size) { + throw new InvalidDataException($"pak file '{path}' is {_fileLength} bytes — smaller than the {PakHeader.Size}-byte header"); + } + + _mmf = MemoryMappedFile.CreateFromFile(path, FileMode.Open, mapName: null, capacity: 0, MemoryMappedFileAccess.Read); + _accessor = _mmf.CreateViewAccessor(0, 0, MemoryMappedFileAccess.Read); + + var headerBytes = new byte[PakHeader.Size]; + _accessor.ReadArray(0, headerBytes, 0, PakHeader.Size); + Header = PakHeader.ReadFrom((ReadOnlySpan)headerBytes); + + long tocBytesTotal = (long)Header.TocCount * PakTocEntry.Size; + if ((long)Header.TocOffset + tocBytesTotal > _fileLength) { + throw new InvalidDataException( + $"pak file '{path}' TOC (offset {Header.TocOffset}, {Header.TocCount} entries) extends past " + + $"the file's actual length ({_fileLength} bytes) — truncated or corrupt file"); + } + + _toc = new PakTocEntry[Header.TocCount]; + var tocBytes = new byte[PakTocEntry.Size]; + long tocPos = (long)Header.TocOffset; + for (int i = 0; i < _toc.Length; i++) { + _accessor.ReadArray(tocPos, tocBytes, 0, PakTocEntry.Size); + _toc[i] = PakTocEntry.ReadFrom((ReadOnlySpan)tocBytes); + tocPos += PakTocEntry.Size; + } + } + + /// True if is present AND its blob's CRC verifies. + public bool ContainsKey(ulong key) { + int index = BinarySearch(key); + if (index < 0) return false; + return VerifyCrc(index); + } + + /// + /// Reads and deserializes the stored under + /// . Returns false (and null via ) + /// if the key is absent OR the blob's CRC fails verification (corruption + /// tripwire — logged once per blob). + /// + public bool TryReadObjectMeshData(ulong key, out ObjectMeshData? data) { + data = null; + int index = BinarySearch(key); + if (index < 0) return false; + if (!VerifyCrc(index)) return false; + + ref readonly var entry = ref _toc[index]; + var bytes = new byte[entry.Length]; + _accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length); + data = ObjectMeshDataSerializer.Read(bytes); + return true; + } + + /// Returns the blob's file offset for alignment assertions in tests. + public long GetBlobOffsetForTest(ulong key) { + int index = BinarySearch(key); + if (index < 0) throw new KeyNotFoundException($"pak key 0x{key:X16} not found"); + return (long)_toc[index].Offset; + } + + /// O(n) linear scan, used only to cross-check the binary search in tests. + public bool DebugLinearScanContainsKey(ulong key) { + for (int i = 0; i < _toc.Length; i++) { + if (_toc[i].Key == key) return VerifyCrc(i); + } + return false; + } + + private bool VerifyCrc(int tocIndex) { + if (_crcVerifiedByTocIndex.TryGetValue(tocIndex, out var cached)) return cached == 1; + + ref readonly var entry = ref _toc[tocIndex]; + var bytes = new byte[entry.Length]; + _accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length); + uint actualCrc = Crc32.Compute(bytes); + bool ok = actualCrc == entry.Crc32; + + _crcVerifiedByTocIndex[tocIndex] = ok ? 1 : 0; + if (!ok && _loggedCorruption.TryAdd(tocIndex, true)) { + Console.Error.WriteLine( + $"[pak-corrupt] key 0x{entry.Key:X16} at offset {entry.Offset} (length {entry.Length}): " + + $"crc mismatch (expected 0x{entry.Crc32:X8}, got 0x{actualCrc:X8}) — treating as missing"); + } + return ok; + } + + private int BinarySearch(ulong key) { + int lo = 0, hi = _toc.Length - 1; + while (lo <= hi) { + int mid = lo + (hi - lo) / 2; + ulong midKey = _toc[mid].Key; + if (midKey == key) return mid; + if (midKey < key) lo = mid + 1; + else hi = mid - 1; + } + return -1; + } + + public void Dispose() { + _accessor.Dispose(); + _mmf.Dispose(); + } +} diff --git a/src/AcDream.Content/Pak/PakWriter.cs b/src/AcDream.Content/Pak/PakWriter.cs new file mode 100644 index 00000000..39618512 --- /dev/null +++ b/src/AcDream.Content/Pak/PakWriter.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace AcDream.Content.Pak; + +/// +/// Streams pak blobs to disk: header placeholder -> 64-byte-aligned blobs -> +/// TOC (sorted by key) -> seek back and finalize the header once +/// tocOffset/tocCount are known. This lets the writer stream blobs without +/// knowing the final count up front (plan: "TOC last"). +/// +public sealed class PakWriter : IDisposable { + private const int Alignment = 64; + + private readonly FileStream _stream; + private readonly PakHeader _headerTemplate; + private readonly List _tocEntries = new(); + private readonly HashSet _seenKeys = new(); + private bool _finished; + + public PakWriter(string path, PakHeader headerTemplate) { + _stream = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None); + _headerTemplate = headerTemplate; + + // Write a header placeholder now; finalized in Finish() once TocOffset/TocCount are known. + _headerTemplate.WriteTo(_stream); + PadToAlignment(); + } + + /// + /// Adds one blob (an serialized via + /// ) under . + /// Blobs are written immediately at the current (64-byte-aligned) stream + /// position; the writer pads after each blob so the NEXT blob also + /// starts aligned. + /// + public void AddBlob(ulong key, ObjectMeshData data) { + ThrowIfFinished(); + if (!_seenKeys.Add(key)) { + throw new ArgumentException($"duplicate pak key 0x{key:X16}", nameof(key)); + } + + long offset = _stream.Position; + using var ms = new MemoryStream(); + ObjectMeshDataSerializer.Write(data, ms); + var bytes = ms.ToArray(); + + _stream.Write(bytes); + PadToAlignment(); + + _tocEntries.Add(new PakTocEntry { + Key = key, + Offset = (ulong)offset, + Length = (uint)bytes.Length, + Crc32 = Content.Pak.Crc32.Compute(bytes), + }); + } + + /// Writes the sorted TOC and finalizes the header. Must be called exactly once. + public void Finish() { + ThrowIfFinished(); + _finished = true; + + var sorted = _tocEntries.OrderBy(e => e.Key).ToList(); + ulong tocOffset = (ulong)_stream.Position; + foreach (var entry in sorted) { + entry.WriteTo(_stream); + } + + var finalHeader = _headerTemplate; + finalHeader.TocOffset = tocOffset; + finalHeader.TocCount = (uint)sorted.Count; + + _stream.Position = 0; + finalHeader.WriteTo(_stream); + _stream.Flush(); + } + + private void PadToAlignment() { + long pos = _stream.Position; + long remainder = pos % Alignment; + if (remainder == 0) return; + long pad = Alignment - remainder; + Span zeros = stackalloc byte[(int)pad]; + zeros.Clear(); + _stream.Write(zeros); + } + + private void ThrowIfFinished() { + if (_finished) throw new InvalidOperationException("PakWriter.Finish() has already been called."); + } + + public void Dispose() { + if (!_finished) { + // Best-effort finalize so a forgotten Finish() call doesn't leave a + // half-written file with a placeholder header claiming 0 entries + // while blobs are on disk — surfaced loudly instead via Finish()'s + // own invariants on next use, but here we simply flush what exists. + try { Finish(); } catch (InvalidOperationException) { /* already finished */ } + } + _stream.Dispose(); + } +} diff --git a/tests/AcDream.Content.Tests/PakRoundTripTests.cs b/tests/AcDream.Content.Tests/PakRoundTripTests.cs new file mode 100644 index 00000000..b46ff594 --- /dev/null +++ b/tests/AcDream.Content.Tests/PakRoundTripTests.cs @@ -0,0 +1,215 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Numerics; +using AcDream.Content.Pak; + +namespace AcDream.Content.Tests; + +public class PakRoundTripTests : IDisposable { + private readonly List _tempFiles = new(); + + private string NewTempPakPath() { + var path = Path.Combine(Path.GetTempPath(), $"acdream-paktest-{Guid.NewGuid():N}.pak"); + _tempFiles.Add(path); + return path; + } + + public void Dispose() { + foreach (var f in _tempFiles) { + try { if (File.Exists(f)) File.Delete(f); } catch { /* best effort cleanup */ } + } + } + + private static ObjectMeshData MakeSyntheticData(uint fileId, int vertexCount) { + var vertices = new VertexPositionNormalTexture[vertexCount]; + for (int i = 0; i < vertexCount; i++) { + vertices[i] = new VertexPositionNormalTexture( + new Vector3(i, i * 2, i * 3), + new Vector3(0, 0, 1), + new Vector2(i * 0.1f, i * 0.2f)); + } + return new ObjectMeshData { + ObjectId = fileId, + IsSetup = false, + Vertices = vertices, + DIDDegrade = fileId + 1, + }; + } + + private static (PakAssetType Type, uint FileId, ObjectMeshData Data)[] MakeBlobSet(int n) { + var result = new (PakAssetType, uint, ObjectMeshData)[n]; + for (int i = 0; i < n; i++) { + var type = (PakAssetType)((i % 3) + 1); + uint fileId = 0x0100_0000u + (uint)i; + result[i] = (type, fileId, MakeSyntheticData(fileId, i + 1)); + } + return result; + } + + private static PakHeader WritePak(string path, (PakAssetType Type, uint FileId, ObjectMeshData Data)[] blobs) { + var header = new PakHeader { + FormatVersion = 1, + PortalIteration = 10, + CellIteration = 20, + HighResIteration = 30, + LanguageIteration = 40, + BakeToolVersion = 1, + }; + using var writer = new PakWriter(path, header); + foreach (var (type, fileId, data) in blobs) { + writer.AddBlob(PakKey.Compose(type, fileId), data); + } + writer.Finish(); + return header; + } + + [Fact] + public void WriteThenOpen_HeaderFieldsMatch() { + var path = NewTempPakPath(); + var blobs = MakeBlobSet(5); + var written = WritePak(path, blobs); + + using var reader = new PakReader(path); + Assert.Equal(written.FormatVersion, reader.Header.FormatVersion); + Assert.Equal(written.PortalIteration, reader.Header.PortalIteration); + Assert.Equal(written.CellIteration, reader.Header.CellIteration); + Assert.Equal(written.HighResIteration, reader.Header.HighResIteration); + Assert.Equal(written.LanguageIteration, reader.Header.LanguageIteration); + Assert.Equal(written.BakeToolVersion, reader.Header.BakeToolVersion); + Assert.Equal((uint)blobs.Length, reader.Header.TocCount); + } + + [Fact] + public void WriteThenOpen_EveryKeyFound() { + var path = NewTempPakPath(); + var blobs = MakeBlobSet(20); + WritePak(path, blobs); + + using var reader = new PakReader(path); + foreach (var (type, fileId, _) in blobs) { + Assert.True(reader.ContainsKey(PakKey.Compose(type, fileId)), + $"key for type={type} fileId=0x{fileId:X8} should be found"); + } + } + + [Fact] + public void WriteThenOpen_BlobsDeserializeToDeepEqualObjects() { + var path = NewTempPakPath(); + var blobs = MakeBlobSet(8); + WritePak(path, blobs); + + using var reader = new PakReader(path); + foreach (var (type, fileId, expected) in blobs) { + var key = PakKey.Compose(type, fileId); + Assert.True(reader.TryReadObjectMeshData(key, out var actual), $"expected key 0x{key:X16} to read successfully"); + ObjectMeshDataEquality.AssertEqual(expected, actual); + } + } + + [Fact] + public void MissingKey_ReturnsFalse() { + var path = NewTempPakPath(); + var blobs = MakeBlobSet(3); + WritePak(path, blobs); + + using var reader = new PakReader(path); + var missingKey = PakKey.Compose(PakAssetType.GfxObjMesh, 0xFFFF_FFFEu); + Assert.False(reader.ContainsKey(missingKey)); + Assert.False(reader.TryReadObjectMeshData(missingKey, out var data)); + Assert.Null(data); + } + + [Fact] + public void CorruptedBlob_CrcMismatch_TreatedAsMissing() { + var path = NewTempPakPath(); + var blobs = MakeBlobSet(4); + WritePak(path, blobs); + + // Flip one byte inside the FIRST blob's region (right after the 64-byte header). + using (var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite)) { + fs.Position = PakHeader.Size + 4; // a few bytes into the first blob's payload + int b = fs.ReadByte(); + fs.Position = PakHeader.Size + 4; + fs.WriteByte((byte)(b ^ 0xFF)); + } + + using var reader = new PakReader(path); + var key = PakKey.Compose(blobs[0].Type, blobs[0].FileId); + Assert.False(reader.TryReadObjectMeshData(key, out var data), + "a CRC-mismatched blob must be treated as missing"); + Assert.Null(data); + } + + [Fact] + public void CorruptedBlob_DoesNotAffectOtherBlobs() { + var path = NewTempPakPath(); + var blobs = MakeBlobSet(4); + WritePak(path, blobs); + + using (var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite)) { + fs.Position = PakHeader.Size + 4; + int b = fs.ReadByte(); + fs.Position = PakHeader.Size + 4; + fs.WriteByte((byte)(b ^ 0xFF)); + } + + using var reader = new PakReader(path); + // blobs[1..] should still read fine. + for (int i = 1; i < blobs.Length; i++) { + var key = PakKey.Compose(blobs[i].Type, blobs[i].FileId); + Assert.True(reader.TryReadObjectMeshData(key, out var actual), $"blob {i} should be unaffected by corruption in blob 0"); + ObjectMeshDataEquality.AssertEqual(blobs[i].Data, actual); + } + } + + [Fact] + public void Blobs_Are64ByteAligned() { + var path = NewTempPakPath(); + var blobs = MakeBlobSet(6); + WritePak(path, blobs); + + using var reader = new PakReader(path); + foreach (var (type, fileId, _) in blobs) { + var offset = reader.GetBlobOffsetForTest(PakKey.Compose(type, fileId)); + Assert.True(offset % 64 == 0, $"blob offset {offset} for type={type} fileId=0x{fileId:X8} must be 64-byte aligned"); + } + } + + [Fact] + public void TocBinarySearch_MatchesLinearScan() { + var path = NewTempPakPath(); + var blobs = MakeBlobSet(50); + WritePak(path, blobs); + + using var reader = new PakReader(path); + var allKeys = blobs.Select(b => PakKey.Compose(b.Type, b.FileId)).ToArray(); + foreach (var key in allKeys) { + bool binarySearchFound = reader.ContainsKey(key); + bool linearScanFound = reader.DebugLinearScanContainsKey(key); + Assert.Equal(linearScanFound, binarySearchFound); + } + + // Also verify a handful of keys NOT present agree between both paths. + var absentKeys = new[] { + PakKey.Compose(PakAssetType.GfxObjMesh, 0xDEAD_0000u), + PakKey.Compose(PakAssetType.SetupMesh, 0xDEAD_0001u), + PakKey.Compose(PakAssetType.EnvCellMesh, 0xDEAD_0002u), + }; + foreach (var key in absentKeys) { + Assert.Equal(reader.DebugLinearScanContainsKey(key), reader.ContainsKey(key)); + Assert.False(reader.ContainsKey(key)); + } + } + + [Fact] + public void EmptyPak_OpensCleanly_NoKeysFound() { + var path = NewTempPakPath(); + WritePak(path, Array.Empty<(PakAssetType, uint, ObjectMeshData)>()); + + using var reader = new PakReader(path); + Assert.Equal(0u, reader.Header.TocCount); + Assert.False(reader.ContainsKey(PakKey.Compose(PakAssetType.GfxObjMesh, 1))); + } +}