diff --git a/.gitignore b/.gitignore index 3f6ee43f..604bb0a7 100644 --- a/.gitignore +++ b/.gitignore @@ -91,3 +91,7 @@ C[€- # Junction to Claude Code per-project memory (Obsidian vault visibility) claude-memory studio-shots/ + +# MP1b acdream-bake output — user-machine artifact, never committed +# (docs/superpowers/plans/2026-07-05-mp1b-pak-and-bake.md, Task 5). +*.pak diff --git a/AcDream.slnx b/AcDream.slnx index 8f761598..c00c5246 100644 --- a/AcDream.slnx +++ b/AcDream.slnx @@ -1,6 +1,7 @@ + diff --git a/src/AcDream.Bake/AcDream.Bake.csproj b/src/AcDream.Bake/AcDream.Bake.csproj new file mode 100644 index 00000000..263db624 --- /dev/null +++ b/src/AcDream.Bake/AcDream.Bake.csproj @@ -0,0 +1,25 @@ + + + + Exe + acdream-bake + net10.0 + enable + enable + latest + true + + + + + + + + + + + + + diff --git a/src/AcDream.Bake/BakeDatCollectionAdapter.cs b/src/AcDream.Bake/BakeDatCollectionAdapter.cs new file mode 100644 index 00000000..a3843b9c --- /dev/null +++ b/src/AcDream.Bake/BakeDatCollectionAdapter.cs @@ -0,0 +1,147 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Collections.ObjectModel; +using System.Diagnostics.CodeAnalysis; +using AcDream.Content; +using DatReaderWriter; +using DatReaderWriter.Enums; +using DatReaderWriter.Lib.IO; + +namespace AcDream.Bake; + +/// +/// Adapts acdream's to +/// for MeshExtractor's constructor. A separate copy from +/// AcDream.App.Rendering.Wb.DatCollectionAdapter (that one is `internal` to +/// AcDream.App and, more importantly, AcDream.App carries the Silk.NET/GL +/// dependency chain the bake tool must never reference — see +/// AcDream.Bake.csproj's "NO Silk.NET" comment). This is plain dat-access +/// glue, not an AC-specific algorithm, so a small duplicated adapter is the +/// right call rather than inventing a shared-but-App-rooted package. +/// +internal sealed class BakeDatCollectionAdapter : IDatReaderWriter { + private readonly DatCollection _dats; + private readonly DatDatabaseWrapper _portal; + private readonly DatDatabaseWrapper _cell; + private readonly DatDatabaseWrapper _highRes; + private readonly DatDatabaseWrapper _language; + private readonly ReadOnlyDictionary _cellRegions; + + public BakeDatCollectionAdapter(DatCollection dats) { + ArgumentNullException.ThrowIfNull(dats); + _dats = dats; + _portal = new DatDatabaseWrapper(dats.Portal); + _cell = new DatDatabaseWrapper(dats.Cell); + _highRes = new DatDatabaseWrapper(dats.HighRes); + _language = new DatDatabaseWrapper(dats.Local); + + var regions = new Dictionary { [0u] = _cell }; + _cellRegions = new ReadOnlyDictionary(regions); + } + + public string SourceDirectory => _dats.Options.DatDirectory ?? string.Empty; + + public IDatDatabase Portal => _portal; + public ReadOnlyDictionary CellRegions => _cellRegions; + public IDatDatabase HighRes => _highRes; + public IDatDatabase Language => _language; + + public ReadOnlyDictionary RegionFileMap => + new ReadOnlyDictionary(new Dictionary()); + + public int PortalIteration => _portal.Iteration; + public int CellIteration => _cell.Iteration; + public int HighResIteration => _highRes.Iteration; + public int LanguageIteration => _language.Iteration; + + public bool TryGetFileBytes(uint regionId, uint fileId, ref byte[] bytes, out int bytesRead) { + return _dats.Cell.TryGetFileBytes(fileId, ref bytes, out bytesRead); + } + + /// Mirrors DefaultDatReaderWriter.ResolveId ordering: HighRes -> Portal -> Language -> Cell. + public IEnumerable ResolveId(uint id) { + var results = new List(); + + void CheckDb(DatDatabaseWrapper wrapper) { + var rawDb = wrapper.RawDatabase; + if (rawDb.Tree.TryGetFile(id, out _)) { + var type = rawDb.TypeFromId(id); + if (type != DBObjType.Unknown) { + results.Add(new IDatReaderWriter.IdResolution(wrapper, type)); + } + } + } + + CheckDb(_highRes); + CheckDb(_portal); + CheckDb(_language); + CheckDb(_cell); + + return results; + } + + public bool TrySave(T obj, int iteration = 0) where T : IDBObj => + throw new NotSupportedException($"{nameof(BakeDatCollectionAdapter)} is read-only."); + + public bool TrySave(uint regionId, T obj, int iteration = 0) where T : IDBObj => + throw new NotSupportedException($"{nameof(BakeDatCollectionAdapter)} is read-only."); + + public void Dispose() { + // The underlying DatCollection is owned by the caller. + } +} + +/// Wraps a as . +internal sealed class DatDatabaseWrapper : IDatDatabase { + private readonly DatDatabase _db; + private readonly ConcurrentDictionary<(Type, uint), IDBObj> _cache = new(); + private readonly object _lock = new(); + + public DatDatabaseWrapper(DatDatabase db) { + ArgumentNullException.ThrowIfNull(db); + _db = db; + } + + internal DatDatabase RawDatabase => _db; + + public DatDatabase Db => _db; + public int Iteration => _db.Iteration?.CurrentIteration ?? 0; + + public IEnumerable GetAllIdsOfType() where T : IDBObj => _db.GetAllIdsOfType(); + + public bool TryGet(uint fileId, [MaybeNullWhen(false)] out T value) where T : IDBObj { + if (_cache.TryGetValue((typeof(T), fileId), out var cached)) { + value = (T)cached; + return true; + } + + lock (_lock) { + if (_db.TryGet(fileId, out value)) { + _cache.TryAdd((typeof(T), fileId), value); + return true; + } + } + + return false; + } + + public bool TryGetFileBytes(uint fileId, [MaybeNullWhen(false)] out byte[] value) { + lock (_lock) { + return _db.TryGetFileBytes(fileId, out value); + } + } + + public bool TryGetFileBytes(uint fileId, ref byte[] bytes, out int bytesRead) { + lock (_lock) { + return _db.TryGetFileBytes(fileId, ref bytes, out bytesRead); + } + } + + public bool TrySave(T obj, int iteration = 0) where T : IDBObj => + throw new NotSupportedException($"{nameof(DatDatabaseWrapper)} is read-only."); + + public void Dispose() { + // The underlying DatDatabase is owned by DatCollection. + } +} diff --git a/src/AcDream.Bake/ConsoleErrorLogger.cs b/src/AcDream.Bake/ConsoleErrorLogger.cs new file mode 100644 index 00000000..a8be4302 --- /dev/null +++ b/src/AcDream.Bake/ConsoleErrorLogger.cs @@ -0,0 +1,38 @@ +using System; +using Microsoft.Extensions.Logging; + +namespace AcDream.Bake; + +/// +/// Minimal writing to stderr with a level prefix. +/// MeshExtractor logs failures via ILogger.LogError/LogWarning (dat-read +/// anomalies, malformed entries) — per the project's "logger injection for +/// silent catches" lesson, the bake tool must NOT wire a NullLogger here or +/// every per-id extraction failure goes silent. A tiny hand-rolled logger +/// avoids pulling in the Microsoft.Extensions.Logging.Console package +/// (not already a transitive dependency) for what is a one-line need. +/// +public sealed class ConsoleErrorLogger : ILogger { + private readonly string _categoryName; + + public ConsoleErrorLogger(string categoryName) { + _categoryName = categoryName; + } + + public IDisposable? BeginScope(TState state) where TState : notnull => null; + + public bool IsEnabled(LogLevel logLevel) => logLevel >= LogLevel.Warning; + + public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { + if (!IsEnabled(logLevel)) return; + var message = formatter(state, exception); + Console.Error.WriteLine($"[{logLevel}] {_categoryName}: {message}"); + if (exception is not null) Console.Error.WriteLine(exception); + } +} + +/// Factory producing instances for any category/type. +public sealed class ConsoleErrorLoggerProvider : ILoggerProvider { + public ILogger CreateLogger(string categoryName) => new ConsoleErrorLogger(categoryName); + public void Dispose() { } +} diff --git a/src/AcDream.Bake/Program.cs b/src/AcDream.Bake/Program.cs new file mode 100644 index 00000000..c8288829 --- /dev/null +++ b/src/AcDream.Bake/Program.cs @@ -0,0 +1,276 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using AcDream.Bake; +using AcDream.Content; +using AcDream.Content.Pak; +using DatReaderWriter; +using DatReaderWriter.DBObjs; +using DatReaderWriter.Options; +using Environment = System.Environment; + +// acdream-bake: offline CLI producing a versioned pak file containing every +// ObjectMeshData the client's decode workers would otherwise produce at +// runtime (GfxObj / Setup / EnvCell mesh+texture payloads). +// +// Drives AcDream.Content.MeshExtractor — the SAME extraction code the live +// client runs — so pak-vs-live equivalence is by construction (verified by +// the Task 6 dat-gated equivalence suite), not by a second hand-written +// extraction path. +// +// Plan: docs/superpowers/plans/2026-07-05-mp1b-pak-and-bake.md, Task 5. + +string? datDir = null; +string? outPath = null; +HashSet? idFilter = null; +HashSet? landblockFilter = null; +int threads = Environment.ProcessorCount; + +for (int i = 0; i < args.Length; i++) { + switch (args[i]) { + case "--dat-dir": + datDir = args.ElementAtOrDefault(++i); + break; + case "--out": + outPath = args.ElementAtOrDefault(++i); + break; + case "--ids": + idFilter = ParseHexList(args.ElementAtOrDefault(++i)); + break; + case "--landblocks": + landblockFilter = ParseHexList(args.ElementAtOrDefault(++i)); + break; + case "--threads": + if (int.TryParse(args.ElementAtOrDefault(++i), out var t) && t > 0) threads = t; + break; + default: + Console.Error.WriteLine($"unrecognized argument: {args[i]}"); + return 2; + } +} + +if (string.IsNullOrWhiteSpace(datDir)) { + Console.Error.WriteLine("usage: acdream-bake --dat-dir [--out ] [--ids 0xId,0xId,...] [--landblocks 0xId,...] [--threads ]"); + return 2; +} + +if (!Directory.Exists(datDir)) { + Console.Error.WriteLine($"error: directory not found: {datDir}"); + return 2; +} + +outPath ??= Path.Combine(datDir, "acdream.pak"); + +Console.WriteLine("acdream-bake"); +Console.WriteLine($"dat dir: {datDir}"); +Console.WriteLine($"out: {outPath}"); +Console.WriteLine($"threads: {threads}"); +Console.WriteLine(); + +var sw = Stopwatch.StartNew(); +using var dats = new DatCollection(datDir, DatAccessType.Read); +using var datReaderWriter = new BakeDatCollectionAdapter(dats); +var extractorLogger = new ConsoleErrorLogger(nameof(MeshExtractor)); + +// A ConcurrentQueue collector for particle-preload meshes MeshExtractor +// side-stages mid-extraction (CollectEmittersFromScript). MeshExtractor is +// documented thread-safe for up to MaxParallelLoads (4) concurrent workers +// sharing one instance and requires this sink argument explicitly — no +// default — so a bake tool can't silently forget it and drop preloads. +// The bake tool treats each side-staged item as its own GfxObjMesh entry, +// deduplicated by id the same way the enumerated id set is. +var sideStaged = new ConcurrentQueue(); +var extractor = new MeshExtractor(datReaderWriter, extractorLogger, data => sideStaged.Enqueue(data)); + +// ---- enumeration --------------------------------------------------------- + +var gfxObjIds = dats.GetAllIdsOfType().ToList(); +var setupIds = dats.GetAllIdsOfType().ToList(); +var envCellIds = EnumerateEnvCellIds(dats, landblockFilter); + +if (idFilter is not null) { + gfxObjIds = gfxObjIds.Where(idFilter.Contains).ToList(); + setupIds = setupIds.Where(idFilter.Contains).ToList(); + envCellIds = envCellIds.Where(idFilter.Contains).ToList(); +} + +var work = new List<(PakAssetType Type, uint FileId)>(gfxObjIds.Count + setupIds.Count + envCellIds.Count); +work.AddRange(gfxObjIds.Select(id => (PakAssetType.GfxObjMesh, id))); +work.AddRange(setupIds.Select(id => (PakAssetType.SetupMesh, id))); +work.AddRange(envCellIds.Select(id => (PakAssetType.EnvCellMesh, id))); + +Console.WriteLine($"enumerated: {gfxObjIds.Count:N0} GfxObj, {setupIds.Count:N0} Setup, {envCellIds.Count:N0} EnvCell " + + $"({work.Count:N0} total)"); +Console.WriteLine(); + +// ---- parallel extract ----------------------------------------------------- + +var results = new ConcurrentBag<(PakAssetType Type, uint FileId, ObjectMeshData Data)>(); +var failures = new ConcurrentBag<(PakAssetType Type, uint FileId, string Reason)>(); +long completed = 0; +var progressLock = new object(); +var lastProgressReport = Stopwatch.StartNew(); + +Parallel.ForEach( + work, + new ParallelOptions { MaxDegreeOfParallelism = threads }, + item => { + var (type, fileId) = item; + try { + ulong extractorId = type == PakAssetType.EnvCellMesh ? fileId | 0x1_0000_0000UL : fileId; + bool isSetup = type == PakAssetType.SetupMesh; + var data = extractor.PrepareMeshData(extractorId, isSetup); + if (data is not null) { + // Extractor echoes back its own resolved ObjectId (which may + // differ in the high bits for EnvCell synthetic-geometry + // requests) — always key the pak entry by the ENUMERATED + // fileId so lookups at read time are predictable. + results.Add((type, fileId, data)); + } + else { + failures.Add((type, fileId, "extractor returned null (no polygons or unresolvable id)")); + } + } + catch (Exception ex) { + // A malformed dat entry skips that id — never fatal to the bake, + // matching the runtime's own per-id try/catch behavior. + failures.Add((type, fileId, ex.Message)); + } + + long done = Interlocked.Increment(ref completed); + lock (progressLock) { + if (lastProgressReport.Elapsed.TotalSeconds >= 5) { + ReportProgress(done, work.Count, failures.Count, sw.Elapsed); + lastProgressReport.Restart(); + } + } + }); + +ReportProgress(completed, work.Count, failures.Count, sw.Elapsed); +Console.WriteLine(); + +// ---- write pak ----------------------------------------------------------- + +var header = new PakHeader { + FormatVersion = 1, + PortalIteration = (uint)dats.Portal.Iteration!.CurrentIteration, + CellIteration = (uint)dats.Cell.Iteration!.CurrentIteration, + HighResIteration = (uint)dats.HighRes.Iteration!.CurrentIteration, + LanguageIteration = (uint)dats.Local.Iteration!.CurrentIteration, + BakeToolVersion = 1, +}; + +var seenSideStagedIds = new HashSet(); +int written = 0, sideStagedWritten = 0, sideStagedDuped = 0; +using (var writer = new PakWriter(outPath, header)) { + foreach (var (type, fileId, data) in results) { + writer.AddBlob(PakKey.Compose(type, fileId), data); + written++; + } + + // Side-staged particle-preload GfxObj meshes: dedup by id against BOTH + // each other and the primary GfxObj set already written (a particle's + // preload GfxObj can also be independently enumerated as a top-level + // GfxObj — MeshExtractor.PrepareMeshData(id, isSetup:false) is + // deterministic, so re-extracting would just waste time, but we must + // never write the SAME pak key twice). + var primaryGfxObjIds = new HashSet(gfxObjIds); + while (sideStaged.TryDequeue(out var staged)) { + uint fileId = (uint)(staged.ObjectId & 0xFFFFFFFFu); + if (primaryGfxObjIds.Contains(fileId) || !seenSideStagedIds.Add(staged.ObjectId)) { + sideStagedDuped++; + continue; + } + writer.AddBlob(PakKey.Compose(PakAssetType.GfxObjMesh, fileId), staged); + sideStagedWritten++; + } + + writer.Finish(); +} + +sw.Stop(); + +var outSize = new FileInfo(outPath).Length; + +Console.WriteLine("=== bake summary ==="); +Console.WriteLine($" GfxObj baked: {results.Count(r => r.Type == PakAssetType.GfxObjMesh):N0}"); +Console.WriteLine($" Setup baked: {results.Count(r => r.Type == PakAssetType.SetupMesh):N0}"); +Console.WriteLine($" EnvCell baked: {results.Count(r => r.Type == PakAssetType.EnvCellMesh):N0}"); +Console.WriteLine($" side-staged baked (particle preload GfxObjs): {sideStagedWritten:N0} ({sideStagedDuped:N0} deduped)"); +Console.WriteLine($" total blobs: {written + sideStagedWritten:N0}"); +Console.WriteLine($" failures: {failures.Count:N0}"); +Console.WriteLine($" elapsed: {sw.Elapsed.TotalSeconds:F1} s"); +Console.WriteLine($" output size: {outSize / 1024.0 / 1024.0:F1} MB"); +Console.WriteLine($" output path: {outPath}"); + +if (failures.Count > 0) { + Console.WriteLine(); + Console.WriteLine($"failures ({failures.Count}):"); + foreach (var (type, fileId, reason) in failures.OrderBy(f => f.FileId).Take(200)) { + Console.WriteLine($" {type,-12} 0x{fileId:X8}: {reason}"); + } + if (failures.Count > 200) Console.WriteLine($" ... and {failures.Count - 200} more"); +} + +return 0; + +// ---- helpers --------------------------------------------------------------- + +static HashSet ParseHexList(string? raw) { + var result = new HashSet(); + if (string.IsNullOrWhiteSpace(raw)) return result; + foreach (var token in raw.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)) { + var hex = token.StartsWith("0x", StringComparison.OrdinalIgnoreCase) ? token[2..] : token; + if (uint.TryParse(hex, System.Globalization.NumberStyles.HexNumber, null, out var value)) { + result.Add(value); + } + else { + Console.Error.WriteLine($"warning: could not parse id '{token}' — skipped"); + } + } + return result; +} + +static void ReportProgress(long done, int total, int failures, TimeSpan elapsed) { + double rate = elapsed.TotalSeconds > 0 ? done / elapsed.TotalSeconds : 0; + double etaSeconds = rate > 0 ? (total - done) / rate : 0; + Console.WriteLine($"[{elapsed:hh\\:mm\\:ss}] baked {done:N0}/{total:N0}, failures={failures:N0}, " + + $"elapsed={elapsed.TotalSeconds:F0}s, ETA={etaSeconds:F0}s"); +} + +/// +/// Enumerates EnvCell ids by walking the cell dat's LandBlockInfo entries +/// (0xFFFE low 16 bits) and, for each, the NumCells-derived cell id range — +/// GetAllIdsOfType<T>() does not cover cell-dat range-based types +/// (DatReaderWriter's documented limitation; see the low-16-bit bucketing +/// idiom in src/AcDream.Cli/Program.cs's CountCellByLow16, and the +/// firstCellId/NumCells hydration idiom in GameWindow.BuildPhysicsDatBundle +/// / BuildInteriorEntitiesForStreaming). +/// +static List EnumerateEnvCellIds(DatCollection dats, HashSet? landblockFilter) { + var landblockInfoIds = new List(); + foreach (var file in dats.Cell.Tree) { + if ((file.Id & 0xFFFFu) != 0xFFFEu) continue; + uint landblockId = file.Id & 0xFFFF0000u; + if (landblockFilter is not null && !landblockFilter.Contains(landblockId)) continue; + landblockInfoIds.Add(file.Id); + } + + var envCellIds = new List(); + foreach (var lbInfoId in landblockInfoIds) { + if (!dats.Cell.TryGet(lbInfoId, out var lbInfo) || lbInfo is null) continue; + if (lbInfo.NumCells == 0) continue; + + uint landblockId = lbInfoId & 0xFFFF0000u; + uint firstCellId = landblockId | 0x0100u; + for (uint offset = 0; offset < lbInfo.NumCells; offset++) { + envCellIds.Add(firstCellId + offset); + } + } + return envCellIds; +}