erik/acdream
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

feat(bake): publish validated pak artifacts atomically

Browse source
This commit is contained in:
Erik 2026-07-24 13:40:58 +02:00
parent 90b378cc70
commit 999201cca7
9 changed files with 362 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

39
src/AcDream.Bake/BakeArtifactValidator.cs Normal file
View file

@ -0,0 +1,39 @@
using AcDream.Content.Pak;
namespace AcDream.Bake;
/// <summary>Strict pre-publication validation for one completed bake artifact.</summary>
public static class BakeArtifactValidator
{
public static void Validate(
string path,
in PakHeader expectedHeader,
int expectedTocCount)
{
using var reader = new PakReader(path);
var actual = reader.Header;
if (actual.FormatVersion != PakFormat.CurrentFormatVersion)
throw new InvalidDataException(
$"bake format version {actual.FormatVersion} does not match " +
$"{PakFormat.CurrentFormatVersion}");
if (actual.BakeToolVersion != PakFormat.CurrentBakeToolVersion)
throw new InvalidDataException(
$"bake tool version {actual.BakeToolVersion} does not match " +
$"{PakFormat.CurrentBakeToolVersion}");
if (actual.PortalIteration != expectedHeader.PortalIteration ||
actual.CellIteration != expectedHeader.CellIteration ||
actual.HighResIteration != expectedHeader.HighResIteration ||
actual.LanguageIteration != expectedHeader.LanguageIteration)
{
throw new InvalidDataException(
"bake DAT iterations do not match the source collection");
}
if (actual.TocCount != checked((uint)expectedTocCount))
throw new InvalidDataException(
$"bake TOC count {actual.TocCount} does not match expected " +
$"{expectedTocCount}");
reader.ValidateComplete();
}
}

61
src/AcDream.Bake/BakeOutputTransaction.cs Normal file
View file

@ -0,0 +1,61 @@
using AcDream.Content.Pak;
namespace AcDream.Bake;
/// <summary>
/// Publishes a bake as one same-volume filesystem transaction. Work and
/// validation happen against a unique adjacent temporary file; failure or
/// cancellation leaves an existing destination untouched.
/// </summary>
public static class BakeOutputTransaction
{
public static TResult WriteValidateAndPublish<TResult>(
string destinationPath,
Func<string, TResult> writeTemporary,
Action<string, TResult> validateTemporary,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(destinationPath);
ArgumentNullException.ThrowIfNull(writeTemporary);
ArgumentNullException.ThrowIfNull(validateTemporary);
string fullDestination = Path.GetFullPath(destinationPath);
string? directory = Path.GetDirectoryName(fullDestination);
if (string.IsNullOrEmpty(directory))
throw new InvalidOperationException("destination has no parent directory");
Directory.CreateDirectory(directory);
string temporaryPath = Path.Combine(
directory,
$".{Path.GetFileName(fullDestination)}.{Guid.NewGuid():N}.tmp");
try
{
cancellationToken.ThrowIfCancellationRequested();
TResult result = writeTemporary(temporaryPath);
cancellationToken.ThrowIfCancellationRequested();
validateTemporary(temporaryPath, result);
cancellationToken.ThrowIfCancellationRequested();
if (File.Exists(fullDestination))
File.Replace(temporaryPath, fullDestination, null, ignoreMetadataErrors: true);
else
File.Move(temporaryPath, fullDestination);
return result;
}
finally
{
try
{
if (File.Exists(temporaryPath))
File.Delete(temporaryPath);
}
catch
{
// Preserve the original exception. An adjacent .tmp is
// recognizable and never mistaken for a published pak.
}
}
}
}

32
src/AcDream.Bake/BakeRunner.cs
View file

@ -24,12 +24,14 @@ public sealed record BakeOptions
/// <summary>Compact result used by the full-scale gate and deterministic tests.</summary>
public sealed record BakeReport
{
public required PakHeader Header { get; init; }
public required int GfxObjKeys { get; init; }
public required int SetupKeys { get; init; }
public required int EnvCellKeys { get; init; }
public required int UniqueEnvCellGeometries { get; init; }
public required int EnvCellAliases { get; init; }
public required int SideStagedKeys { get; init; }
public int SideStagedDuplicateKeys { get; init; }
public required int PhysicalBlobs { get; init; }
public required int TotalKeys { get; init; }
public required int Failures { get; init; }
@ -66,12 +68,29 @@ public static class BakeRunner
if (options.Threads <= 0)
throw new ArgumentOutOfRangeException(nameof(options), "thread count must be positive");
options.CancellationToken.ThrowIfCancellationRequested();
var report = BakeOutputTransaction.WriteValidateAndPublish(
options.OutPath,
temporaryPath => RunCore(
options with { OutPath = temporaryPath },
options.OutPath),
(temporaryPath, result) => BakeArtifactValidator.Validate(
temporaryPath,
result.Header,
result.TotalKeys),
options.CancellationToken);
PrintSummary(report, options.OutPath);
return report;
}
private static BakeReport RunCore(BakeOptions options, string publishedOutputPath)
{
var cancellationToken = options.CancellationToken;
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine("acdream-bake");
Console.WriteLine($"dat dir: {options.DatDir}");
Console.WriteLine($"out: {options.OutPath}");
Console.WriteLine($"out: {publishedOutputPath}");
Console.WriteLine($"threads: {options.Threads}");
Console.WriteLine();
@ -147,7 +166,7 @@ public static class BakeRunner
CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration,
HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration,
LanguageIteration = (uint)dats.Local.Iteration!.CurrentIteration,
BakeToolVersion = 1,
BakeToolVersion = PakFormat.CurrentBakeToolVersion,
};
var writtenKeys = new HashSet<ulong>();
@ -339,6 +358,7 @@ public static class BakeRunner
var report = new BakeReport
{
Header = header,
GfxObjKeys = gfxObjWritten,
SetupKeys = setupWritten,
EnvCellKeys = envCellKeysWritten,
@ -354,7 +374,7 @@ public static class BakeRunner
PeakWorkingSetBytes = Process.GetCurrentProcess().PeakWorkingSet64,
};
PrintSummary(report, sideStagedDuped, options.OutPath);
report = report with { SideStagedDuplicateKeys = sideStagedDuped };
PrintFailures(failures);
return report;
}
@ -392,7 +412,7 @@ public static class BakeRunner
lastProgressReport.Restart();
}
private static void PrintSummary(BakeReport report, int sideStagedDuped, string outputPath)
private static void PrintSummary(BakeReport report, string outputPath)
{
Console.WriteLine();
Console.WriteLine("=== bake summary ===");
@ -404,7 +424,7 @@ public static class BakeRunner
Console.WriteLine($" EnvCell dedup: {report.EnvCellDedupRatio:F1}x");
Console.WriteLine(
$" side-staged keys: {report.SideStagedKeys:N0} " +
$"({sideStagedDuped:N0} duplicate keys)");
$"({report.SideStagedDuplicateKeys:N0} duplicate keys)");
Console.WriteLine($" physical blobs: {report.PhysicalBlobs:N0}");
Console.WriteLine($" total keys: {report.TotalKeys:N0}");
Console.WriteLine($" failures: {report.Failures:N0}");

9
src/AcDream.Content/Pak/PakFormat.cs
View file

@ -15,6 +15,13 @@ public static class PakFormat {
/// with an accompanying reader migration path.
/// </summary>
public const uint CurrentFormatVersion = 1;
/// <summary>
/// Identity of the bake algorithm that produced the payloads. Version 2
/// introduces content-identity EnvCell blobs with ordinary TOC aliases.
/// The binary format remains version 1.
/// </summary>
public const uint CurrentBakeToolVersion = 2;
}
/// <summary>
@ -29,7 +36,7 @@ public static class PakFormat {
/// 20 4 languageIteration
/// 24 8 tocOffset (u64)
/// 32 4 tocCount (u32)
/// 36 4 bakeToolVersion = 1
/// 36 4 bakeToolVersion
/// 40 24 reserved (zero)
/// </code>
/// Spec: docs/superpowers/plans/2026-07-05-mp1b-pak-and-bake.md "Format v1 (normative)".

60
src/AcDream.Content/Pak/PakReader.cs
View file

@ -96,11 +96,7 @@ public sealed class PakReader : IDisposable {
// blob region must sit fully between the header and the TOC, and
// its offset must survive the (long) cast the accessor needs.
ref readonly var entry = ref _toc[i];
bool invalid =
entry.Offset < PakHeader.Size ||
entry.Offset > long.MaxValue ||
entry.Offset + entry.Length > Header.TocOffset ||
entry.Offset + entry.Length > (ulong)_fileLength;
bool invalid = !IsEntryRangeValid(entry);
if (invalid) {
_entryVerdictByTocIndex[i] = 0;
LogCorruptionOnce(i, $"TOC entry out of bounds (offset={entry.Offset}, length={entry.Length}, " +
@ -177,6 +173,47 @@ public sealed class PakReader : IDisposable {
return _toc[index];
}
/// <summary>
/// Validates the complete immutable artifact before an offline bake
/// publishes it. This is intentionally stricter than runtime lookup:
/// TOC order/ranges and the CRC of every unique physical blob must all be
/// valid. Aliases are checked once per shared receipt.
/// </summary>
public void ValidateComplete() {
var validatedReceipts = new HashSet<(ulong Offset, uint Length, uint Crc32)>();
ulong previousKey = 0;
for (int i = 0; i < _toc.Length; i++) {
ref readonly var entry = ref _toc[i];
if (i > 0 && entry.Key <= previousKey) {
throw new InvalidDataException(
$"pak TOC is not strictly sorted at entry {i}: " +
$"0x{entry.Key:X16} follows 0x{previousKey:X16}");
}
previousKey = entry.Key;
if (!IsEntryRangeValid(entry)) {
throw new InvalidDataException(
$"pak TOC entry 0x{entry.Key:X16} has an invalid range " +
$"(offset={entry.Offset}, length={entry.Length}, " +
$"file={_fileLength}, toc@{Header.TocOffset})");
}
var receipt = (entry.Offset, entry.Length, entry.Crc32);
if (!validatedReceipts.Add(receipt))
continue;
var bytes = new byte[entry.Length];
_accessor.ReadArray((long)entry.Offset, bytes, 0, (int)entry.Length);
uint actualCrc = Crc32.Compute(bytes);
if (actualCrc != entry.Crc32) {
throw new InvalidDataException(
$"pak blob 0x{entry.Key:X16} failed CRC validation: " +
$"expected 0x{entry.Crc32:X8}, got 0x{actualCrc:X8}");
}
}
}
/// <summary>O(n) linear scan, used only to cross-check the binary search in tests.</summary>
public bool DebugLinearScanContainsKey(ulong key) {
for (int i = 0; i < _toc.Length; i++) {
@ -222,6 +259,19 @@ public sealed class PakReader : IDisposable {
return -1;
}
private bool IsEntryRangeValid(in PakTocEntry entry) {
ulong fileLength = (ulong)_fileLength;
if (entry.Offset < PakHeader.Size ||
entry.Offset > long.MaxValue ||
entry.Offset > Header.TocOffset ||
entry.Offset > fileLength) {
return false;
}
return entry.Length <= Header.TocOffset - entry.Offset &&
entry.Length <= fileLength - entry.Offset;
}
public void Dispose() {
_accessor.Dispose();
_mmf.Dispose();

1
src/AcDream.Content/Pak/PakWriter.cs
View file

@ -30,6 +30,7 @@ public sealed class PakWriter : IDisposable {
// (the default-0 footgun; PakReader rejects any version but
// PakFormat.CurrentFormatVersion).
_headerTemplate.FormatVersion = PakFormat.CurrentFormatVersion;
_headerTemplate.BakeToolVersion = PakFormat.CurrentBakeToolVersion;
// Write a header placeholder now; finalized in Finish() once TocOffset/TocCount are known.
_headerTemplate.WriteTo(_stream);

170
tests/AcDream.Bake.Tests/BakeOutputTransactionTests.cs Normal file
View file

@ -0,0 +1,170 @@
using AcDream.Content;
using AcDream.Content.Pak;
namespace AcDream.Bake.Tests;
public sealed class BakeOutputTransactionTests : IDisposable
{
private readonly string _directory =
Path.Combine(Path.GetTempPath(), $"acdream-bake-transaction-{Guid.NewGuid():N}");
public BakeOutputTransactionTests() => Directory.CreateDirectory(_directory);
public void Dispose()
{
try
{
if (Directory.Exists(_directory))
Directory.Delete(_directory, recursive: true);
}
catch
{
// Best-effort test cleanup.
}
}
[Fact]
public void Publish_ReplacesExistingDestinationOnlyAfterValidation()
{
string destination = Path.Combine(_directory, "content.pak");
File.WriteAllText(destination, "old");
int result = BakeOutputTransaction.WriteValidateAndPublish(
destination,
temporary =>
{
File.WriteAllText(temporary, "new");
return 42;
},
(temporary, value) =>
{
Assert.Equal(42, value);
Assert.Equal("new", File.ReadAllText(temporary));
Assert.Equal("old", File.ReadAllText(destination));
});
Assert.Equal(42, result);
Assert.Equal("new", File.ReadAllText(destination));
AssertNoTemporaryFiles();
}
[Theory]
[InlineData(true)]
[InlineData(false)]
public void Failure_PreservesExistingDestinationAndCleansTemporary(bool failDuringWrite)
{
string destination = Path.Combine(_directory, "content.pak");
File.WriteAllText(destination, "old");
Assert.Throws<InvalidOperationException>(
() => BakeOutputTransaction.WriteValidateAndPublish(
destination,
temporary =>
{
File.WriteAllText(temporary, "partial");
if (failDuringWrite)
throw new InvalidOperationException("write failed");
return 1;
},
(_, _) => throw new InvalidOperationException("validation failed")));
Assert.Equal("old", File.ReadAllText(destination));
AssertNoTemporaryFiles();
}
[Fact]
public void CancellationBeforeWrite_PreservesExistingDestination()
{
string destination = Path.Combine(_directory, "content.pak");
File.WriteAllText(destination, "old");
using var cancellation = new CancellationTokenSource();
cancellation.Cancel();
Assert.Throws<OperationCanceledException>(
() => BakeOutputTransaction.WriteValidateAndPublish(
destination,
temporary =>
{
File.WriteAllText(temporary, "new");
return 1;
},
(_, _) => { },
cancellation.Token));
Assert.Equal("old", File.ReadAllText(destination));
AssertNoTemporaryFiles();
}
[Fact]
public void ArtifactValidator_RejectsStaleBakeToolVersion()
{
string pak = Path.Combine(_directory, "stale.pak");
var expected = Header();
using (var writer = new PakWriter(pak, expected))
{
writer.AddBlob(
PakKey.Compose(PakAssetType.GfxObjMesh, 1),
new ObjectMeshData { ObjectId = 1 });
writer.Finish();
}
using (var stream = new FileStream(pak, FileMode.Open, FileAccess.ReadWrite))
{
stream.Position = 36;
Span<byte> stale = stackalloc byte[4];
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(stale, 1);
stream.Write(stale);
}
var exception = Assert.Throws<InvalidDataException>(
() => BakeArtifactValidator.Validate(pak, expected, expectedTocCount: 1));
Assert.Contains("bake tool version", exception.Message);
}
[Fact]
public void ArtifactValidator_RejectsWrongDatIterationAndCorruptBlob()
{
string pak = Path.Combine(_directory, "content.pak");
var expected = Header();
using (var writer = new PakWriter(pak, expected))
{
writer.AddBlob(
PakKey.Compose(PakAssetType.GfxObjMesh, 1),
new ObjectMeshData { ObjectId = 1 });
writer.Finish();
}
var wrongIteration = expected;
wrongIteration.CellIteration++;
Assert.Throws<InvalidDataException>(
() => BakeArtifactValidator.Validate(pak, wrongIteration, expectedTocCount: 1));
long offset;
using (var reader = new PakReader(pak))
offset = reader.GetBlobOffsetForTest(
PakKey.Compose(PakAssetType.GfxObjMesh, 1));
using (var stream = new FileStream(pak, FileMode.Open, FileAccess.ReadWrite))
{
stream.Position = offset;
int value = stream.ReadByte();
stream.Position = offset;
stream.WriteByte((byte)(value ^ 0xFF));
}
Assert.Throws<InvalidDataException>(
() => BakeArtifactValidator.Validate(pak, expected, expectedTocCount: 1));
}
private PakHeader Header() =>
new()
{
PortalIteration = 10,
CellIteration = 20,
HighResIteration = 30,
LanguageIteration = 40,
BakeToolVersion = PakFormat.CurrentBakeToolVersion,
};
private void AssertNoTemporaryFiles() =>
Assert.Empty(Directory.EnumerateFiles(_directory, ".*.tmp"));
}

2
tests/AcDream.Content.Tests/PakEquivalenceTests.cs
View file

@ -92,7 +92,7 @@ public sealed class PakEquivalenceTests {
CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration,
HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration,
LanguageIteration = (uint)dats.Local.Iteration!.CurrentIteration,
BakeToolVersion = 1,
BakeToolVersion = PakFormat.CurrentBakeToolVersion,
};
using (var writer = new PakWriter(pakPath, header)) {
foreach (var ((type, fileId), data) in golden) {

2
tests/AcDream.Content.Tests/PakRoundTripTests.cs
View file

@ -55,7 +55,7 @@ public class PakRoundTripTests : IDisposable {
CellIteration = 20,
HighResIteration = 30,
LanguageIteration = 40,
BakeToolVersion = 1,
BakeToolVersion = PakFormat.CurrentBakeToolVersion,
};
using var writer = new PakWriter(path, header);
foreach (var (type, fileId, data) in blobs) {