erik/openDecal
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix DecalDat to work with real AC DAT files

Browse source 
The DAT file reader had several bugs inherited from the old C++ reference
code, which targeted an older format version. Verified and fixed against
real client_portal.dat and client_cell_1.dat files:

- Fix header offset: BTree root is at 0x160, not 0x148 (file size field)
- Fix BTree entry size: 24 bytes (flags+id+offset+size+timestamp), not 12
- Fix sector-chain node reading: BTree nodes span multiple sectors via
  linked-list headers; must assemble node data across sector boundaries
- Fix DatStreamImpl.Read() BSTR handling: use Buffer.BlockCopy to match
  C++ SysAllocStringByteLen instead of Marshal.PtrToStringAnsi
- Fix DatStreamImpl.ReadBinary() pointer lifetime: inline fixed block to
  keep destination buffer pinned during Marshal.Copy
- Document LoadFilters() dependency on parameterized COM properties in
  IDecalCore.Configuration that need IDispatch to call correctly

Add smoke test project (13/13 tests pass against real DAT files).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
erik 2026-02-08 23:12:04 +01:00
parent c0d1135431
commit f0b6fedc9b
5 changed files with 356 additions and 56 deletions
Download patch file Download diff file Expand all files Collapse all files

11
Managed/Decal.DecalDat.Tests/Decal.DecalDat.Tests.csproj Normal file
View file

@ -0,0 +1,11 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<AssemblyName>Decal.DecalDat.Tests</AssemblyName>
<RootNamespace>Decal.DecalDat.Tests</RootNamespace>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\Decal.DecalDat\Decal.DecalDat.csproj" />
<ProjectReference Include="..\Decal.Interop.Dat\Decal.Interop.Dat.csproj" />
</ItemGroup>
</Project>

219
Managed/Decal.DecalDat.Tests/Program.cs Normal file
View file

@ -0,0 +1,219 @@
using System;
using System.IO;
using System.Runtime.InteropServices;
using Decal.Interop.Dat;
namespace Decal.DecalDat.Tests
{
class Program
{
// File IDs from FileService.Startup() — decimal values converted to hex
static readonly (uint id, string name)[] PortalFiles = new[]
{
(0x0E00000Eu, "SpellTable"), // 234881038
(0x0E00000Fu, "ComponentTable"), // 234881039
(0x0E000003u, "VitalFormulaTable"), // 234881027
(0x0E000004u, "SkillTable"), // 234881028
(0x25000006u, "AttributeTable"), // 620756998
(0x25000007u, "VitalTable"), // 620756999
};
static int _pass, _fail;
static void Main(string[] args)
{
string acPath = args.Length > 0
? args[0]
: @"C:\Turbine\Asheron's Call";
string portalPath = Path.Combine(acPath, "client_portal.dat");
string cellPath = Path.Combine(acPath, "client_cell_1.dat");
Console.WriteLine("=== DecalDat Smoke Test ===");
Console.WriteLine($"Platform: {(IntPtr.Size == 4 ? "x86" : "x64")}");
Console.WriteLine();
TestDatFile("portal", portalPath, 1024);
TestDatFile("cell", cellPath, 256);
TestPortalKnownFiles(portalPath);
TestStreamReadRestart(portalPath);
TestStreamReadBinary(portalPath);
Console.WriteLine();
Console.WriteLine($"=== Results: {_pass} passed, {_fail} failed ===");
}
static void TestDatFile(string label, string path, int sectorSize)
{
Console.WriteLine($"[TEST] Open {label}.dat ({path})");
if (!File.Exists(path))
{
Fail($" File not found: {path}");
return;
}
try
{
using (var dat = new DatFile(path, sectorSize))
{
Pass($" Opened successfully (sector size {sectorSize})");
}
}
catch (Exception ex)
{
Fail($" Failed to open: {ex.Message}");
}
}
static void TestPortalKnownFiles(string portalPath)
{
if (!File.Exists(portalPath)) return;
Console.WriteLine();
Console.WriteLine("[TEST] Read known portal.dat file IDs");
try
{
using (var dat = new DatFile(portalPath, 1024))
{
foreach (var (id, name) in PortalFiles)
{
try
{
var entry = dat.GetFile(id);
if (entry.Size > 0)
Pass($" 0x{id:X8} ({name}): {entry.Size:N0} bytes");
else
Fail($" 0x{id:X8} ({name}): size is 0");
}
catch (FileNotFoundException)
{
Fail($" 0x{id:X8} ({name}): NOT FOUND");
}
catch (Exception ex)
{
Fail($" 0x{id:X8} ({name}): ERROR - {ex.GetType().Name}: {ex.Message}");
}
}
}
}
catch (Exception ex)
{
Fail($" Failed to open DAT: {ex.Message}");
}
}
static void TestStreamReadRestart(string portalPath)
{
if (!File.Exists(portalPath)) return;
Console.WriteLine();
Console.WriteLine("[TEST] DatStream: Read, Restart, Read again (consistency)");
using (var dat = new DatFile(portalPath, 1024))
{
var entry = dat.GetFile(0x0E00000E); // SpellTable
var stream = new DatStreamImpl();
stream.Load(entry);
int size = stream.Size;
if (size <= 0) { Fail("Size is 0"); return; }
// Read first 64 bytes via ReadBinary
int readSize = Math.Min(64, size);
byte[] first = new byte[readSize];
stream.ReadBinary(readSize, ref first[0]);
// Restart and read again
stream.Restart();
byte[] second = new byte[readSize];
stream.ReadBinary(readSize, ref second[0]);
bool match = true;
for (int i = 0; i < readSize; i++)
if (first[i] != second[i]) { match = false; break; }
if (match)
Pass($"Read {readSize} bytes, restarted, read again — identical");
else
Fail("Data mismatch after Restart!");
stream.Restart();
if (stream.Tell == 0)
Pass("Tell resets to 0 after Restart");
else
Fail($"Tell is {stream.Tell} after Restart (expected 0)");
}
}
static void TestStreamReadBinary(string portalPath)
{
if (!File.Exists(portalPath)) return;
Console.WriteLine();
Console.WriteLine("[TEST] DatStream: ReadBinary consistency + Tell tracking");
using (var dat = new DatFile(portalPath, 1024))
{
var entry = dat.GetFile(0x0E000004); // SkillTable
var stream = new DatStreamImpl();
stream.Load(entry);
int size = stream.Size;
int readSize = Math.Min(256, size);
// Read in two chunks
int chunk1 = readSize / 2;
int chunk2 = readSize - chunk1;
byte[] viaChunks = new byte[readSize];
stream.ReadBinary(chunk1, ref viaChunks[0]);
if (stream.Tell == chunk1)
Pass($"Tell is {chunk1} after reading {chunk1} bytes");
else
Fail($"Tell is {stream.Tell} (expected {chunk1})");
// Read second chunk into offset position
byte[] temp = new byte[chunk2];
stream.ReadBinary(chunk2, ref temp[0]);
Array.Copy(temp, 0, viaChunks, chunk1, chunk2);
// Restart and read all at once
stream.Restart();
byte[] viaFull = new byte[readSize];
stream.ReadBinary(readSize, ref viaFull[0]);
bool match = true;
for (int i = 0; i < readSize; i++)
if (viaChunks[i] != viaFull[i]) { match = false; break; }
if (match)
Pass($"Chunked read ({chunk1}+{chunk2}) matches full read ({readSize})");
else
Fail("Chunked vs full read mismatch!");
if (stream.Tell == readSize)
Pass($"Tell is {readSize} after reading {readSize} bytes");
else
Fail($"Tell is {stream.Tell} (expected {readSize})");
}
}
static void Pass(string msg)
{
_pass++;
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine($" PASS {msg}");
Console.ResetColor();
}
static void Fail(string msg)
{
_fail++;
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine($" FAIL {msg}");
Console.ResetColor();
}
}
}

114
Managed/Decal.DecalDat/DatFile.cs
View file

@ -1,8 +1,11 @@
using System;
using System.IO;
using System.IO.MemoryMappedFiles;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
[assembly: InternalsVisibleTo("Decal.DecalDat.Tests")]
namespace Decal.DecalDat
{
/// <summary>
@ -11,17 +14,38 @@ namespace Decal.DecalDat
/// </summary>
internal sealed class DatFile : IDisposable
{
private const int FileCount = 62;
private const int FileCountOffset = 0x03E;
private const int RootDirPtrOffset = 0x148;
// DAT header layout (starting at 0x140):
// 0x140: "BT" magic (0x5442)
// 0x144: block/sector size
// 0x148: file size
// 0x14C: data set type (1=portal, 2=cell)
// 0x150: data subset
// 0x154: free head
// 0x158: free tail
// 0x15C: free count
// 0x160: BTree root directory offset
private const int RootDirPtrOffset = 0x160;
// Directory entry layout (packed):
// DWORD subdirs[62] = 248 bytes
// DWORD fileCount = 4 bytes
// FileEntry files[62] = 744 bytes (62 * 12)
// Total = 996 bytes
private const int SubdirsSize = FileCount * 4;
private const int FileEntrySize = 12; // 3 DWORDs: ID, Offset, Size
// BTree node data layout (within sector chain, after skipping sector headers):
// DWORD branches[62] = 248 bytes (child pointers; 0 = leaf)
// DWORD entryCount = 4 bytes
// FileEntry entries[62] = 1488 bytes (62 * 24)
// Total node data = 1740 bytes
//
// Nodes are stored in sector chains: each sector's first 4 bytes = next sector
// offset, with usable data in bytes [4..sectorSize-1].
//
// Each FileEntry (24 bytes):
// DWORD flags (+0)
// DWORD objectId (+4) — the file ID used for lookup
// DWORD fileOffset (+8) — sector offset to file data
// DWORD fileSize (+12) — size in bytes
// UINT64 timestamp (+16) — Windows FILETIME
private const int MaxEntries = 62;
private const int BranchCount = MaxEntries; // 62 child pointers
private const int BranchesSize = BranchCount * 4; // 248
private const int FileEntrySize = 24; // flags(4) + id(4) + offset(4) + size(4) + timestamp(8)
private const int MaxNodeDataSize = BranchesSize + 4 + MaxEntries * FileEntrySize; // 1740
private MemoryMappedFile _mmf;
private MemoryMappedViewAccessor _accessor;
@ -44,30 +68,76 @@ namespace Decal.DecalDat
return FindFile(rootDirOffset, fileId);
}
private DatFileEntry FindFile(uint dirOffset, uint fileId)
/// <summary>
/// Reads BTree node data from a sector chain into a contiguous byte array.
/// Each sector's first 4 bytes is the next-sector pointer; data follows.
/// </summary>
private byte[] ReadNodeData(uint sectorOffset)
{
// Read file count from directory
uint fileCount = _accessor.ReadUInt32(dirOffset + SubdirsSize);
var data = new byte[MaxNodeDataSize];
int dataPerSector = _sectorSize - 4;
int copied = 0;
uint currentSector = sectorOffset;
while (copied < MaxNodeDataSize && currentSector != 0 && currentSector < (uint)_fileLength)
{
int toCopy = Math.Min(dataPerSector, MaxNodeDataSize - copied);
// Bounds check: ensure we don't read past end of file
long readStart = currentSector + 4;
if (readStart + toCopy > _fileLength)
toCopy = (int)(_fileLength - readStart);
if (toCopy <= 0) break;
_accessor.ReadArray(readStart, data, copied, toCopy);
copied += toCopy;
if (copied < MaxNodeDataSize)
currentSector = _accessor.ReadUInt32(currentSector);
else
break;
}
return data;
}
private static uint ReadUInt32(byte[] data, int offset)
{
return BitConverter.ToUInt32(data, offset);
}
private DatFileEntry FindFile(uint sectorOffset, uint fileId)
{
if (sectorOffset >= (uint)_fileLength)
throw new FileNotFoundException($"BTree branch 0x{sectorOffset:X8} is out of bounds (file=0x{_fileLength:X})");
byte[] node = ReadNodeData(sectorOffset);
// Parse node: branches[62] (248 bytes) + entryCount (4 bytes) + entries
uint entryCount = ReadUInt32(node, BranchesSize);
if (entryCount > MaxEntries)
throw new InvalidDataException($"BTree node at 0x{sectorOffset:X8} has invalid entry count {entryCount}");
// Binary search through sorted file entries
int lo = 0, hi = (int)fileCount - 1;
int lo = 0, hi = (int)entryCount - 1;
while (lo <= hi)
{
int mid = (lo + hi) / 2;
long entryOffset = dirOffset + SubdirsSize + 4 + (mid * FileEntrySize);
uint entryId = _accessor.ReadUInt32(entryOffset);
int entryOffset = BranchesSize + 4 + (mid * FileEntrySize);
// Entry layout: flags(+0), objectId(+4), fileOffset(+8), fileSize(+12), timestamp(+16)
uint entryId = ReadUInt32(node, entryOffset + 4);
if (entryId == fileId)
{
uint offset = _accessor.ReadUInt32(entryOffset + 4);
uint size = _accessor.ReadUInt32(entryOffset + 8);
uint offset = ReadUInt32(node, entryOffset + 8);
uint size = ReadUInt32(node, entryOffset + 12);
return new DatFileEntry(this, offset, size);
}
else if (fileId < entryId)
{
// Recurse into left subdirectory
uint subdir = _accessor.ReadUInt32(dirOffset + (uint)(mid * 4));
// Recurse into left child branch
uint subdir = ReadUInt32(node, mid * 4);
if (subdir != 0)
{
try
@ -87,8 +157,8 @@ namespace Decal.DecalDat
}
}
// Check rightmost subdirectory
uint rightSubdir = _accessor.ReadUInt32(dirOffset + (uint)(lo * 4));
// Check rightmost child branch
uint rightSubdir = ReadUInt32(node, lo * 4);
if (rightSubdir != 0)
{
return FindFile(rightSubdir, fileId);

33
Managed/Decal.DecalDat/DatServiceImpl.cs
View file

@ -55,22 +55,25 @@ namespace Decal.DecalDat
private void LoadFilters()
{
try
{
var config = _decalCore.Configuration;
if (config == null) return;
// Look up "FileFilters" collection in Decal configuration
// The C++ code iterates an IDecalEnum to read filter definitions
// Each filter has: Prefix (string), Cache (bool), ComClass (CLSID)
// The C++ original calls:
// m_pDecal->get_Configuration(L"FileFilters", GUID_NULL, &pEnum)
// then iterates the IDecalEnum reading:
// pEnum->get_ComClass(&clsid)
// pEnum->get_Property(L"Prefix", &vPrefix) // protocol name, e.g. "portal"
// pEnum->get_Property(L"Cache", &vCache) // 0 or 1
//
// For now, register the known built-in filters that ship with Decal.
// When Decal.Core is implemented, this will read from live config.
}
catch
{
// Config may not be available yet
}
// However, IDecalCore.Configuration and IDecalEnum.Property are
// parameterized COM properties that the decompiler flattened to
// simple getters. They need to be called via IDispatch to pass
// the category name / property name arguments.
//
// Filters are registered externally (by plugins/installer in the
// registry under HKLM\SOFTWARE\Decal\FileFilters\{CLSID}) and
// will be loaded once Decal.Core exposes the Configuration enum.
//
// DecalDat still works without filters — callers get raw IDatStream
// objects. Filters are an optional layer that parses the stream into
// typed objects (e.g., spell tables, character data).
}
public void BeforePlugins()

33
Managed/Decal.DecalDat/DatStreamImpl.cs
View file

@ -38,8 +38,14 @@ namespace Decal.DecalDat
var buf = new byte[Bytes];
_file.Read(buf, 0, Bytes);
// Copy to unmanaged buffer starting at ref Buffer
Marshal.Copy(buf, 0, GetBufferPtr(ref Buffer), Bytes);
// Pin the destination buffer for the duration of the copy
unsafe
{
fixed (byte* p = &Buffer)
{
Marshal.Copy(buf, 0, new IntPtr(p), Bytes);
}
}
}
public string Read(int Bytes)
@ -49,22 +55,13 @@ namespace Decal.DecalDat
var buf = new byte[Bytes];
int read = _file.Read(buf, 0, Bytes);
// Return as binary BSTR (same as SysAllocStringByteLen in C++)
unsafe
{
fixed (byte* p = buf)
{
return Marshal.PtrToStringAnsi(new IntPtr(p), read);
}
}
}
private static unsafe IntPtr GetBufferPtr(ref byte buffer)
{
fixed (byte* p = &buffer)
{
return new IntPtr(p);
}
// Match C++ SysAllocStringByteLen: pack raw bytes into WCHAR pairs.
// When the CLR marshals this string to BSTR, the raw bytes are preserved
// byte-for-byte (each char = 2 raw bytes, little-endian).
int charCount = (read + 1) / 2;
var chars = new char[charCount];
Buffer.BlockCopy(buf, 0, chars, 0, read);
return new string(chars, 0, charCount);
}
}
}