erik/acdream
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(pipeline): MP1b - PakWriter + mmap PakReader

Browse source 
TDD: PakRoundTripTests written first, confirmed a compile failure
against the not-yet-existing PakWriter/PakReader types.

PakWriter streams [header placeholder][64-byte-aligned blobs][TOC sorted
by key], then seeks back and finalizes the header once TocOffset/TocCount
are known (plan: "TOC last" so the writer doesn't need blob count
up front). PakReader mmaps the whole file once, loads the TOC into a
sorted array for O(log n) binary-search lookup, and verifies each blob's
CRC-32 LAZILY on first access (cached per-index so a corrupt blob logs
exactly once and is thereafter always treated as missing rather than
handing back garbage bytes). No locks anywhere — the mapped memory is
immutable for the reader's lifetime by construction.

CRC-32 implemented directly (standard IEEE 802.3 table-driven variant)
rather than adding a System.IO.Hashing package reference, keeping
AcDream.Content's dependency surface minimal per its existing
"no GL binaries" intent.

9 tests green: header round-trip, every-key-found, blob deep-equality
via the Task 3 comparator, missing-key returns false, corrupted-blob
(single flipped byte) is detected and treated as missing while leaving
sibling blobs unaffected, 64-byte blob alignment, and TOC binary search
cross-checked against a linear scan for both present and absent keys.
This commit is contained in:
Erik 2026-07-05 21:21:58 +02:00
parent a5ba435839
commit 981025d58b
4 changed files with 495 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

36
src/AcDream.Content/Pak/Crc32.cs Normal file
View file

@ -0,0 +1,36 @@
using System;
namespace AcDream.Content.Pak;
/// <summary>
/// Standard CRC-32 (IEEE 802.3 polynomial 0xEDB88320, the same variant used
/// by zip/gzip/PNG). Implemented directly rather than pulling in
/// System.IO.Hashing to keep AcDream.Content's dependency surface minimal
/// (mirrors the "no GL binaries, keep it lean" intent of the project — see
/// AcDream.Content.csproj's package comments). Used as the pak TOC's
/// per-blob corruption tripwire.
/// </summary>
public static class Crc32 {
private static readonly uint[] Table = BuildTable();
private static uint[] BuildTable() {
const uint polynomial = 0xEDB88320u;
var table = new uint[256];
for (uint i = 0; i < 256; i++) {
uint c = i;
for (int k = 0; k < 8; k++) {
c = (c & 1) != 0 ? polynomial ^ (c >> 1) : c >> 1;
}
table[i] = c;
}
return table;
}
public static uint Compute(ReadOnlySpan<byte> data) {
uint crc = 0xFFFFFFFFu;
foreach (var b in data) {
crc = Table[(crc ^ b) & 0xFF] ^ (crc >> 8);
}
return crc ^ 0xFFFFFFFFu;
}
}

139
src/AcDream.Content/Pak/PakReader.cs Normal file
View file

@ -0,0 +1,139 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.IO.MemoryMappedFiles;
namespace AcDream.Content.Pak;
/// <summary>
/// Zero-copy mmap reader for an acdream pak file. One <see cref="MemoryMappedFile"/>
/// for the whole file's lifetime; TOC is loaded once at construction into a
/// key-sorted array for O(log n) binary-search lookup. Blob CRCs are
/// verified LAZILY on first access of each blob (not eagerly at open) and
/// cached so a corrupt blob is only logged once; a corrupted blob is
/// thereafter always treated as "missing" rather than returning garbage.
///
/// No locks anywhere: the underlying mapped memory is immutable for the
/// lifetime of this reader (acdream never mutates a pak file while it's
/// open for reading — MP1c's cutover writes a NEW pak and swaps it in), so
/// concurrent reads from multiple threads are safe by construction.
/// </summary>
public sealed class PakReader : IDisposable {
private readonly MemoryMappedFile _mmf;
private readonly MemoryMappedViewAccessor _accessor;
private readonly long _fileLength;
private readonly PakTocEntry[] _toc; // sorted ascending by Key
/// <summary>-1 = not yet checked, 0 = crc mismatch (corrupt), 1 = crc ok.</summary>
private readonly ConcurrentDictionary<int, int> _crcVerifiedByTocIndex = new();
private readonly ConcurrentDictionary<int, bool> _loggedCorruption = new();
public PakHeader Header { get; }
public PakReader(string path) {
_fileLength = new FileInfo(path).Length;
if (_fileLength < PakHeader.Size) {
throw new InvalidDataException($"pak file '{path}' is {_fileLength} bytes — smaller than the {PakHeader.Size}-byte header");
}
_mmf = MemoryMappedFile.CreateFromFile(path, FileMode.Open, mapName: null, capacity: 0, MemoryMappedFileAccess.Read);
_accessor = _mmf.CreateViewAccessor(0, 0, MemoryMappedFileAccess.Read);
var headerBytes = new byte[PakHeader.Size];
_accessor.ReadArray(0, headerBytes, 0, PakHeader.Size);
Header = PakHeader.ReadFrom((ReadOnlySpan<byte>)headerBytes);
long tocBytesTotal = (long)Header.TocCount * PakTocEntry.Size;
if ((long)Header.TocOffset + tocBytesTotal > _fileLength) {
throw new InvalidDataException(
$"pak file '{path}' TOC (offset {Header.TocOffset}, {Header.TocCount} entries) extends past " +
$"the file's actual length ({_fileLength} bytes) — truncated or corrupt file");
}
_toc = new PakTocEntry[Header.TocCount];
var tocBytes = new byte[PakTocEntry.Size];
long tocPos = (long)Header.TocOffset;
for (int i = 0; i < _toc.Length; i++) {
_accessor.ReadArray(tocPos, tocBytes, 0, PakTocEntry.Size);
_toc[i] = PakTocEntry.ReadFrom((ReadOnlySpan<byte>)tocBytes);
tocPos += PakTocEntry.Size;
}
}
/// <summary>True if <paramref name="key"/> is present AND its blob's CRC verifies.</summary>
public bool ContainsKey(ulong key) {
int index = BinarySearch(key);
if (index < 0) return false;
return VerifyCrc(index);
}
/// <summary>
/// Reads and deserializes the <see cref="ObjectMeshData"/> stored under
/// <paramref name="key"/>. Returns false (and null via <paramref name="data"/>)
/// if the key is absent OR the blob's CRC fails verification (corruption
/// tripwire — logged once per blob).
/// </summary>
public bool TryReadObjectMeshData(ulong key, out ObjectMeshData? data) {
data = null;
int index = BinarySearch(key);
if (index < 0) return false;
if (!VerifyCrc(index)) return false;
ref readonly var entry = ref _toc[index];
var bytes = new byte[entry.Length];
_accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length);
data = ObjectMeshDataSerializer.Read(bytes);
return true;
}
/// <summary>Returns the blob's file offset for alignment assertions in tests.</summary>
public long GetBlobOffsetForTest(ulong key) {
int index = BinarySearch(key);
if (index < 0) throw new KeyNotFoundException($"pak key 0x{key:X16} not found");
return (long)_toc[index].Offset;
}
/// <summary>O(n) linear scan, used only to cross-check the binary search in tests.</summary>
public bool DebugLinearScanContainsKey(ulong key) {
for (int i = 0; i < _toc.Length; i++) {
if (_toc[i].Key == key) return VerifyCrc(i);
}
return false;
}
private bool VerifyCrc(int tocIndex) {
if (_crcVerifiedByTocIndex.TryGetValue(tocIndex, out var cached)) return cached == 1;
ref readonly var entry = ref _toc[tocIndex];
var bytes = new byte[entry.Length];
_accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length);
uint actualCrc = Crc32.Compute(bytes);
bool ok = actualCrc == entry.Crc32;
_crcVerifiedByTocIndex[tocIndex] = ok ? 1 : 0;
if (!ok && _loggedCorruption.TryAdd(tocIndex, true)) {
Console.Error.WriteLine(
$"[pak-corrupt] key 0x{entry.Key:X16} at offset {entry.Offset} (length {entry.Length}): " +
$"crc mismatch (expected 0x{entry.Crc32:X8}, got 0x{actualCrc:X8}) — treating as missing");
}
return ok;
}
private int BinarySearch(ulong key) {
int lo = 0, hi = _toc.Length - 1;
while (lo <= hi) {
int mid = lo + (hi - lo) / 2;
ulong midKey = _toc[mid].Key;
if (midKey == key) return mid;
if (midKey < key) lo = mid + 1;
else hi = mid - 1;
}
return -1;
}
public void Dispose() {
_accessor.Dispose();
_mmf.Dispose();
}
}

105
src/AcDream.Content/Pak/PakWriter.cs Normal file
View file

@ -0,0 +1,105 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace AcDream.Content.Pak;
/// <summary>
/// Streams pak blobs to disk: header placeholder -> 64-byte-aligned blobs ->
/// TOC (sorted by key) -> seek back and finalize the header once
/// tocOffset/tocCount are known. This lets the writer stream blobs without
/// knowing the final count up front (plan: "TOC last").
/// </summary>
public sealed class PakWriter : IDisposable {
private const int Alignment = 64;
private readonly FileStream _stream;
private readonly PakHeader _headerTemplate;
private readonly List<PakTocEntry> _tocEntries = new();
private readonly HashSet<ulong> _seenKeys = new();
private bool _finished;
public PakWriter(string path, PakHeader headerTemplate) {
_stream = new FileStream(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None);
_headerTemplate = headerTemplate;
// Write a header placeholder now; finalized in Finish() once TocOffset/TocCount are known.
_headerTemplate.WriteTo(_stream);
PadToAlignment();
}
/// <summary>
/// Adds one blob (an <see cref="ObjectMeshData"/> serialized via
/// <see cref="ObjectMeshDataSerializer"/>) under <paramref name="key"/>.
/// Blobs are written immediately at the current (64-byte-aligned) stream
/// position; the writer pads after each blob so the NEXT blob also
/// starts aligned.
/// </summary>
public void AddBlob(ulong key, ObjectMeshData data) {
ThrowIfFinished();
if (!_seenKeys.Add(key)) {
throw new ArgumentException($"duplicate pak key 0x{key:X16}", nameof(key));
}
long offset = _stream.Position;
using var ms = new MemoryStream();
ObjectMeshDataSerializer.Write(data, ms);
var bytes = ms.ToArray();
_stream.Write(bytes);
PadToAlignment();
_tocEntries.Add(new PakTocEntry {
Key = key,
Offset = (ulong)offset,
Length = (uint)bytes.Length,
Crc32 = Content.Pak.Crc32.Compute(bytes),
});
}
/// <summary>Writes the sorted TOC and finalizes the header. Must be called exactly once.</summary>
public void Finish() {
ThrowIfFinished();
_finished = true;
var sorted = _tocEntries.OrderBy(e => e.Key).ToList();
ulong tocOffset = (ulong)_stream.Position;
foreach (var entry in sorted) {
entry.WriteTo(_stream);
}
var finalHeader = _headerTemplate;
finalHeader.TocOffset = tocOffset;
finalHeader.TocCount = (uint)sorted.Count;
_stream.Position = 0;
finalHeader.WriteTo(_stream);
_stream.Flush();
}
private void PadToAlignment() {
long pos = _stream.Position;
long remainder = pos % Alignment;
if (remainder == 0) return;
long pad = Alignment - remainder;
Span<byte> zeros = stackalloc byte[(int)pad];
zeros.Clear();
_stream.Write(zeros);
}
private void ThrowIfFinished() {
if (_finished) throw new InvalidOperationException("PakWriter.Finish() has already been called.");
}
public void Dispose() {
if (!_finished) {
// Best-effort finalize so a forgotten Finish() call doesn't leave a
// half-written file with a placeholder header claiming 0 entries
// while blobs are on disk — surfaced loudly instead via Finish()'s
// own invariants on next use, but here we simply flush what exists.
try { Finish(); } catch (InvalidOperationException) { /* already finished */ }
}
_stream.Dispose();
}
}

215
tests/AcDream.Content.Tests/PakRoundTripTests.cs Normal file
View file

@ -0,0 +1,215 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Numerics;
using AcDream.Content.Pak;
namespace AcDream.Content.Tests;
public class PakRoundTripTests : IDisposable {
private readonly List<string> _tempFiles = new();
private string NewTempPakPath() {
var path = Path.Combine(Path.GetTempPath(), $"acdream-paktest-{Guid.NewGuid():N}.pak");
_tempFiles.Add(path);
return path;
}
public void Dispose() {
foreach (var f in _tempFiles) {
try { if (File.Exists(f)) File.Delete(f); } catch { /* best effort cleanup */ }
}
}
private static ObjectMeshData MakeSyntheticData(uint fileId, int vertexCount) {
var vertices = new VertexPositionNormalTexture[vertexCount];
for (int i = 0; i < vertexCount; i++) {
vertices[i] = new VertexPositionNormalTexture(
new Vector3(i, i * 2, i * 3),
new Vector3(0, 0, 1),
new Vector2(i * 0.1f, i * 0.2f));
}
return new ObjectMeshData {
ObjectId = fileId,
IsSetup = false,
Vertices = vertices,
DIDDegrade = fileId + 1,
};
}
private static (PakAssetType Type, uint FileId, ObjectMeshData Data)[] MakeBlobSet(int n) {
var result = new (PakAssetType, uint, ObjectMeshData)[n];
for (int i = 0; i < n; i++) {
var type = (PakAssetType)((i % 3) + 1);
uint fileId = 0x0100_0000u + (uint)i;
result[i] = (type, fileId, MakeSyntheticData(fileId, i + 1));
}
return result;
}
private static PakHeader WritePak(string path, (PakAssetType Type, uint FileId, ObjectMeshData Data)[] blobs) {
var header = new PakHeader {
FormatVersion = 1,
PortalIteration = 10,
CellIteration = 20,
HighResIteration = 30,
LanguageIteration = 40,
BakeToolVersion = 1,
};
using var writer = new PakWriter(path, header);
foreach (var (type, fileId, data) in blobs) {
writer.AddBlob(PakKey.Compose(type, fileId), data);
}
writer.Finish();
return header;
}
[Fact]
public void WriteThenOpen_HeaderFieldsMatch() {
var path = NewTempPakPath();
var blobs = MakeBlobSet(5);
var written = WritePak(path, blobs);
using var reader = new PakReader(path);
Assert.Equal(written.FormatVersion, reader.Header.FormatVersion);
Assert.Equal(written.PortalIteration, reader.Header.PortalIteration);
Assert.Equal(written.CellIteration, reader.Header.CellIteration);
Assert.Equal(written.HighResIteration, reader.Header.HighResIteration);
Assert.Equal(written.LanguageIteration, reader.Header.LanguageIteration);
Assert.Equal(written.BakeToolVersion, reader.Header.BakeToolVersion);
Assert.Equal((uint)blobs.Length, reader.Header.TocCount);
}
[Fact]
public void WriteThenOpen_EveryKeyFound() {
var path = NewTempPakPath();
var blobs = MakeBlobSet(20);
WritePak(path, blobs);
using var reader = new PakReader(path);
foreach (var (type, fileId, _) in blobs) {
Assert.True(reader.ContainsKey(PakKey.Compose(type, fileId)),
$"key for type={type} fileId=0x{fileId:X8} should be found");
}
}
[Fact]
public void WriteThenOpen_BlobsDeserializeToDeepEqualObjects() {
var path = NewTempPakPath();
var blobs = MakeBlobSet(8);
WritePak(path, blobs);
using var reader = new PakReader(path);
foreach (var (type, fileId, expected) in blobs) {
var key = PakKey.Compose(type, fileId);
Assert.True(reader.TryReadObjectMeshData(key, out var actual), $"expected key 0x{key:X16} to read successfully");
ObjectMeshDataEquality.AssertEqual(expected, actual);
}
}
[Fact]
public void MissingKey_ReturnsFalse() {
var path = NewTempPakPath();
var blobs = MakeBlobSet(3);
WritePak(path, blobs);
using var reader = new PakReader(path);
var missingKey = PakKey.Compose(PakAssetType.GfxObjMesh, 0xFFFF_FFFEu);
Assert.False(reader.ContainsKey(missingKey));
Assert.False(reader.TryReadObjectMeshData(missingKey, out var data));
Assert.Null(data);
}
[Fact]
public void CorruptedBlob_CrcMismatch_TreatedAsMissing() {
var path = NewTempPakPath();
var blobs = MakeBlobSet(4);
WritePak(path, blobs);
// Flip one byte inside the FIRST blob's region (right after the 64-byte header).
using (var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite)) {
fs.Position = PakHeader.Size + 4; // a few bytes into the first blob's payload
int b = fs.ReadByte();
fs.Position = PakHeader.Size + 4;
fs.WriteByte((byte)(b ^ 0xFF));
}
using var reader = new PakReader(path);
var key = PakKey.Compose(blobs[0].Type, blobs[0].FileId);
Assert.False(reader.TryReadObjectMeshData(key, out var data),
"a CRC-mismatched blob must be treated as missing");
Assert.Null(data);
}
[Fact]
public void CorruptedBlob_DoesNotAffectOtherBlobs() {
var path = NewTempPakPath();
var blobs = MakeBlobSet(4);
WritePak(path, blobs);
using (var fs = new FileStream(path, FileMode.Open, FileAccess.ReadWrite)) {
fs.Position = PakHeader.Size + 4;
int b = fs.ReadByte();
fs.Position = PakHeader.Size + 4;
fs.WriteByte((byte)(b ^ 0xFF));
}
using var reader = new PakReader(path);
// blobs[1..] should still read fine.
for (int i = 1; i < blobs.Length; i++) {
var key = PakKey.Compose(blobs[i].Type, blobs[i].FileId);
Assert.True(reader.TryReadObjectMeshData(key, out var actual), $"blob {i} should be unaffected by corruption in blob 0");
ObjectMeshDataEquality.AssertEqual(blobs[i].Data, actual);
}
}
[Fact]
public void Blobs_Are64ByteAligned() {
var path = NewTempPakPath();
var blobs = MakeBlobSet(6);
WritePak(path, blobs);
using var reader = new PakReader(path);
foreach (var (type, fileId, _) in blobs) {
var offset = reader.GetBlobOffsetForTest(PakKey.Compose(type, fileId));
Assert.True(offset % 64 == 0, $"blob offset {offset} for type={type} fileId=0x{fileId:X8} must be 64-byte aligned");
}
}
[Fact]
public void TocBinarySearch_MatchesLinearScan() {
var path = NewTempPakPath();
var blobs = MakeBlobSet(50);
WritePak(path, blobs);
using var reader = new PakReader(path);
var allKeys = blobs.Select(b => PakKey.Compose(b.Type, b.FileId)).ToArray();
foreach (var key in allKeys) {
bool binarySearchFound = reader.ContainsKey(key);
bool linearScanFound = reader.DebugLinearScanContainsKey(key);
Assert.Equal(linearScanFound, binarySearchFound);
}
// Also verify a handful of keys NOT present agree between both paths.
var absentKeys = new[] {
PakKey.Compose(PakAssetType.GfxObjMesh, 0xDEAD_0000u),
PakKey.Compose(PakAssetType.SetupMesh, 0xDEAD_0001u),
PakKey.Compose(PakAssetType.EnvCellMesh, 0xDEAD_0002u),
};
foreach (var key in absentKeys) {
Assert.Equal(reader.DebugLinearScanContainsKey(key), reader.ContainsKey(key));
Assert.False(reader.ContainsKey(key));
}
}
[Fact]
public void EmptyPak_OpensCleanly_NoKeysFound() {
var path = NewTempPakPath();
WritePak(path, Array.Empty<(PakAssetType, uint, ObjectMeshData)>());
using var reader = new PakReader(path);
Assert.Equal(0u, reader.Header.TocCount);
Assert.False(reader.ContainsKey(PakKey.Compose(PakAssetType.GfxObjMesh, 1)));
}
}