Fix DecalDat to work with real AC DAT files

The DAT file reader had several bugs inherited from the old C++ reference
code, which targeted an older format version. Verified and fixed against
real client_portal.dat and client_cell_1.dat files:

- Fix header offset: BTree root is at 0x160, not 0x148 (file size field)
- Fix BTree entry size: 24 bytes (flags+id+offset+size+timestamp), not 12
- Fix sector-chain node reading: BTree nodes span multiple sectors via
  linked-list headers; must assemble node data across sector boundaries
- Fix DatStreamImpl.Read() BSTR handling: use Buffer.BlockCopy to match
  C++ SysAllocStringByteLen instead of Marshal.PtrToStringAnsi
- Fix DatStreamImpl.ReadBinary() pointer lifetime: inline fixed block to
  keep destination buffer pinned during Marshal.Copy
- Document LoadFilters() dependency on parameterized COM properties in
  IDecalCore.Configuration that need IDispatch to call correctly

Add smoke test project (13/13 tests pass against real DAT files).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Managed/Decal.DecalDat.Tests/Decal.DecalDat.Tests.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <AssemblyName>Decal.DecalDat.Tests</AssemblyName>
+    <RootNamespace>Decal.DecalDat.Tests</RootNamespace>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Decal.DecalDat\Decal.DecalDat.csproj" />
+    <ProjectReference Include="..\Decal.Interop.Dat\Decal.Interop.Dat.csproj" />
+  </ItemGroup>
+</Project>

Managed/Decal.DecalDat.Tests/Program.cs

@@ -0,0 +1,219 @@
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using Decal.Interop.Dat;
+
+namespace Decal.DecalDat.Tests
+{
+    class Program
+    {
+        // File IDs from FileService.Startup() — decimal values converted to hex
+        static readonly (uint id, string name)[] PortalFiles = new[]
+        {
+            (0x0E00000Eu, "SpellTable"),       // 234881038
+            (0x0E00000Fu, "ComponentTable"),    // 234881039
+            (0x0E000003u, "VitalFormulaTable"), // 234881027
+            (0x0E000004u, "SkillTable"),        // 234881028
+            (0x25000006u, "AttributeTable"),    // 620756998
+            (0x25000007u, "VitalTable"),        // 620756999
+        };
+
+        static int _pass, _fail;
+
+        static void Main(string[] args)
+        {
+            string acPath = args.Length > 0
+                ? args[0]
+                : @"C:\Turbine\Asheron's Call";
+
+            string portalPath = Path.Combine(acPath, "client_portal.dat");
+            string cellPath = Path.Combine(acPath, "client_cell_1.dat");
+
+            Console.WriteLine("=== DecalDat Smoke Test ===");
+            Console.WriteLine($"Platform: {(IntPtr.Size == 4 ? "x86" : "x64")}");
+            Console.WriteLine();
+
+            TestDatFile("portal", portalPath, 1024);
+            TestDatFile("cell", cellPath, 256);
+            TestPortalKnownFiles(portalPath);
+            TestStreamReadRestart(portalPath);
+            TestStreamReadBinary(portalPath);
+
+            Console.WriteLine();
+            Console.WriteLine($"=== Results: {_pass} passed, {_fail} failed ===");
+        }
+
+        static void TestDatFile(string label, string path, int sectorSize)
+        {
+            Console.WriteLine($"[TEST] Open {label}.dat ({path})");
+            if (!File.Exists(path))
+            {
+                Fail($"  File not found: {path}");
+                return;
+            }
+
+            try
+            {
+                using (var dat = new DatFile(path, sectorSize))
+                {
+                    Pass($"  Opened successfully (sector size {sectorSize})");
+                }
+            }
+            catch (Exception ex)
+            {
+                Fail($"  Failed to open: {ex.Message}");
+            }
+        }
+
+        static void TestPortalKnownFiles(string portalPath)
+        {
+            if (!File.Exists(portalPath)) return;
+
+            Console.WriteLine();
+            Console.WriteLine("[TEST] Read known portal.dat file IDs");
+
+            try
+            {
+                using (var dat = new DatFile(portalPath, 1024))
+                {
+                    foreach (var (id, name) in PortalFiles)
+                    {
+                        try
+                        {
+                            var entry = dat.GetFile(id);
+                            if (entry.Size > 0)
+                                Pass($"  0x{id:X8} ({name}): {entry.Size:N0} bytes");
+                            else
+                                Fail($"  0x{id:X8} ({name}): size is 0");
+                        }
+                        catch (FileNotFoundException)
+                        {
+                            Fail($"  0x{id:X8} ({name}): NOT FOUND");
+                        }
+                        catch (Exception ex)
+                        {
+                            Fail($"  0x{id:X8} ({name}): ERROR - {ex.GetType().Name}: {ex.Message}");
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Fail($"  Failed to open DAT: {ex.Message}");
+            }
+        }
+
+        static void TestStreamReadRestart(string portalPath)
+        {
+            if (!File.Exists(portalPath)) return;
+
+            Console.WriteLine();
+            Console.WriteLine("[TEST] DatStream: Read, Restart, Read again (consistency)");
+
+            using (var dat = new DatFile(portalPath, 1024))
+            {
+                var entry = dat.GetFile(0x0E00000E); // SpellTable
+                var stream = new DatStreamImpl();
+                stream.Load(entry);
+
+                int size = stream.Size;
+                if (size <= 0) { Fail("Size is 0"); return; }
+
+                // Read first 64 bytes via ReadBinary
+                int readSize = Math.Min(64, size);
+                byte[] first = new byte[readSize];
+                stream.ReadBinary(readSize, ref first[0]);
+
+                // Restart and read again
+                stream.Restart();
+                byte[] second = new byte[readSize];
+                stream.ReadBinary(readSize, ref second[0]);
+
+                bool match = true;
+                for (int i = 0; i < readSize; i++)
+                    if (first[i] != second[i]) { match = false; break; }
+
+                if (match)
+                    Pass($"Read {readSize} bytes, restarted, read again — identical");
+                else
+                    Fail("Data mismatch after Restart!");
+
+                stream.Restart();
+                if (stream.Tell == 0)
+                    Pass("Tell resets to 0 after Restart");
+                else
+                    Fail($"Tell is {stream.Tell} after Restart (expected 0)");
+            }
+        }
+
+        static void TestStreamReadBinary(string portalPath)
+        {
+            if (!File.Exists(portalPath)) return;
+
+            Console.WriteLine();
+            Console.WriteLine("[TEST] DatStream: ReadBinary consistency + Tell tracking");
+
+            using (var dat = new DatFile(portalPath, 1024))
+            {
+                var entry = dat.GetFile(0x0E000004); // SkillTable
+                var stream = new DatStreamImpl();
+                stream.Load(entry);
+
+                int size = stream.Size;
+                int readSize = Math.Min(256, size);
+
+                // Read in two chunks
+                int chunk1 = readSize / 2;
+                int chunk2 = readSize - chunk1;
+
+                byte[] viaChunks = new byte[readSize];
+                stream.ReadBinary(chunk1, ref viaChunks[0]);
+
+                if (stream.Tell == chunk1)
+                    Pass($"Tell is {chunk1} after reading {chunk1} bytes");
+                else
+                    Fail($"Tell is {stream.Tell} (expected {chunk1})");
+
+                // Read second chunk into offset position
+                byte[] temp = new byte[chunk2];
+                stream.ReadBinary(chunk2, ref temp[0]);
+                Array.Copy(temp, 0, viaChunks, chunk1, chunk2);
+
+                // Restart and read all at once
+                stream.Restart();
+                byte[] viaFull = new byte[readSize];
+                stream.ReadBinary(readSize, ref viaFull[0]);
+
+                bool match = true;
+                for (int i = 0; i < readSize; i++)
+                    if (viaChunks[i] != viaFull[i]) { match = false; break; }
+
+                if (match)
+                    Pass($"Chunked read ({chunk1}+{chunk2}) matches full read ({readSize})");
+                else
+                    Fail("Chunked vs full read mismatch!");
+
+                if (stream.Tell == readSize)
+                    Pass($"Tell is {readSize} after reading {readSize} bytes");
+                else
+                    Fail($"Tell is {stream.Tell} (expected {readSize})");
+            }
+        }
+
+        static void Pass(string msg)
+        {
+            _pass++;
+            Console.ForegroundColor = ConsoleColor.Green;
+            Console.WriteLine($"  PASS {msg}");
+            Console.ResetColor();
+        }
+
+        static void Fail(string msg)
+        {
+            _fail++;
+            Console.ForegroundColor = ConsoleColor.Red;
+            Console.WriteLine($"  FAIL {msg}");
+            Console.ResetColor();
+        }
+    }
+}

Managed/Decal.DecalDat/DatFile.cs

@@ -1,8 +1,11 @@
 using System;
 using System.IO;
 using System.IO.MemoryMappedFiles;
+using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
+[assembly: InternalsVisibleTo("Decal.DecalDat.Tests")]
+
 namespace Decal.DecalDat
 {
     /// <summary>
@@ -11,17 +14,38 @@ namespace Decal.DecalDat
     /// </summary>
     internal sealed class DatFile : IDisposable
     {
-        private const int FileCount = 62;
+        // DAT header layout (starting at 0x140):
-        private const int FileCountOffset = 0x03E;
+        //   0x140: "BT" magic (0x5442)
-        private const int RootDirPtrOffset = 0x148;
+        //   0x144: block/sector size
+        //   0x148: file size
+        //   0x14C: data set type (1=portal, 2=cell)
+        //   0x150: data subset
+        //   0x154: free head
+        //   0x158: free tail
+        //   0x15C: free count
+        //   0x160: BTree root directory offset
+        private const int RootDirPtrOffset = 0x160;
 
-        // Directory entry layout (packed):
+        // BTree node data layout (within sector chain, after skipping sector headers):
-        //   DWORD subdirs[62]    = 248 bytes
+        //   DWORD branches[62]           = 248 bytes  (child pointers; 0 = leaf)
-        //   DWORD fileCount      =   4 bytes
+        //   DWORD entryCount             =   4 bytes
-        //   FileEntry files[62]  = 744 bytes (62 * 12)
+        //   FileEntry entries[62]        = 1488 bytes (62 * 24)
-        //   Total                = 996 bytes
+        //   Total node data              = 1740 bytes
-        private const int SubdirsSize = FileCount * 4;
+        //
-        private const int FileEntrySize = 12; // 3 DWORDs: ID, Offset, Size
+        // Nodes are stored in sector chains: each sector's first 4 bytes = next sector
+        // offset, with usable data in bytes [4..sectorSize-1].
+        //
+        // Each FileEntry (24 bytes):
+        //   DWORD flags      (+0)
+        //   DWORD objectId   (+4)   — the file ID used for lookup
+        //   DWORD fileOffset (+8)   — sector offset to file data
+        //   DWORD fileSize   (+12)  — size in bytes
+        //   UINT64 timestamp (+16)  — Windows FILETIME
+        private const int MaxEntries = 62;
+        private const int BranchCount = MaxEntries; // 62 child pointers
+        private const int BranchesSize = BranchCount * 4; // 248
+        private const int FileEntrySize = 24; // flags(4) + id(4) + offset(4) + size(4) + timestamp(8)
+        private const int MaxNodeDataSize = BranchesSize + 4 + MaxEntries * FileEntrySize; // 1740
 
         private MemoryMappedFile _mmf;
         private MemoryMappedViewAccessor _accessor;
@@ -44,30 +68,76 @@ namespace Decal.DecalDat
             return FindFile(rootDirOffset, fileId);
         }
 
-        private DatFileEntry FindFile(uint dirOffset, uint fileId)
+        /// <summary>
+        /// Reads BTree node data from a sector chain into a contiguous byte array.
+        /// Each sector's first 4 bytes is the next-sector pointer; data follows.
+        /// </summary>
+        private byte[] ReadNodeData(uint sectorOffset)
         {
-            // Read file count from directory
+            var data = new byte[MaxNodeDataSize];
-            uint fileCount = _accessor.ReadUInt32(dirOffset + SubdirsSize);
+            int dataPerSector = _sectorSize - 4;
+            int copied = 0;
+            uint currentSector = sectorOffset;
+
+            while (copied < MaxNodeDataSize && currentSector != 0 && currentSector < (uint)_fileLength)
+            {
+                int toCopy = Math.Min(dataPerSector, MaxNodeDataSize - copied);
+                // Bounds check: ensure we don't read past end of file
+                long readStart = currentSector + 4;
+                if (readStart + toCopy > _fileLength)
+                    toCopy = (int)(_fileLength - readStart);
+                if (toCopy <= 0) break;
+
+                _accessor.ReadArray(readStart, data, copied, toCopy);
+                copied += toCopy;
+
+                if (copied < MaxNodeDataSize)
+                    currentSector = _accessor.ReadUInt32(currentSector);
+                else
+                    break;
+            }
+
+            return data;
+        }
+
+        private static uint ReadUInt32(byte[] data, int offset)
+        {
+            return BitConverter.ToUInt32(data, offset);
+        }
+
+        private DatFileEntry FindFile(uint sectorOffset, uint fileId)
+        {
+            if (sectorOffset >= (uint)_fileLength)
+                throw new FileNotFoundException($"BTree branch 0x{sectorOffset:X8} is out of bounds (file=0x{_fileLength:X})");
+
+            byte[] node = ReadNodeData(sectorOffset);
+
+            // Parse node: branches[62] (248 bytes) + entryCount (4 bytes) + entries
+            uint entryCount = ReadUInt32(node, BranchesSize);
+
+            if (entryCount > MaxEntries)
+                throw new InvalidDataException($"BTree node at 0x{sectorOffset:X8} has invalid entry count {entryCount}");
 
             // Binary search through sorted file entries
-            int lo = 0, hi = (int)fileCount - 1;
+            int lo = 0, hi = (int)entryCount - 1;
 
             while (lo <= hi)
             {
                 int mid = (lo + hi) / 2;
-                long entryOffset = dirOffset + SubdirsSize + 4 + (mid * FileEntrySize);
+                int entryOffset = BranchesSize + 4 + (mid * FileEntrySize);
-                uint entryId = _accessor.ReadUInt32(entryOffset);
+                // Entry layout: flags(+0), objectId(+4), fileOffset(+8), fileSize(+12), timestamp(+16)
+                uint entryId = ReadUInt32(node, entryOffset + 4);
 
                 if (entryId == fileId)
                 {
-                    uint offset = _accessor.ReadUInt32(entryOffset + 4);
+                    uint offset = ReadUInt32(node, entryOffset + 8);
-                    uint size = _accessor.ReadUInt32(entryOffset + 8);
+                    uint size = ReadUInt32(node, entryOffset + 12);
                     return new DatFileEntry(this, offset, size);
                 }
                 else if (fileId < entryId)
                 {
-                    // Recurse into left subdirectory
+                    // Recurse into left child branch
-                    uint subdir = _accessor.ReadUInt32(dirOffset + (uint)(mid * 4));
+                    uint subdir = ReadUInt32(node, mid * 4);
                     if (subdir != 0)
                     {
                         try
@@ -87,8 +157,8 @@ namespace Decal.DecalDat
                 }
             }
 
-            // Check rightmost subdirectory
+            // Check rightmost child branch
-            uint rightSubdir = _accessor.ReadUInt32(dirOffset + (uint)(lo * 4));
+            uint rightSubdir = ReadUInt32(node, lo * 4);
             if (rightSubdir != 0)
             {
                 return FindFile(rightSubdir, fileId);

Managed/Decal.DecalDat/DatServiceImpl.cs

@@ -55,22 +55,25 @@ namespace Decal.DecalDat
 
         private void LoadFilters()
         {
-            try
+            // The C++ original calls:
-            {
+            //   m_pDecal->get_Configuration(L"FileFilters", GUID_NULL, &pEnum)
-                var config = _decalCore.Configuration;
+            // then iterates the IDecalEnum reading:
-                if (config == null) return;
+            //   pEnum->get_ComClass(&clsid)
-
+            //   pEnum->get_Property(L"Prefix", &vPrefix)   // protocol name, e.g. "portal"
-                // Look up "FileFilters" collection in Decal configuration
+            //   pEnum->get_Property(L"Cache", &vCache)      // 0 or 1
-                // The C++ code iterates an IDecalEnum to read filter definitions
+            //
-                // Each filter has: Prefix (string), Cache (bool), ComClass (CLSID)
+            // However, IDecalCore.Configuration and IDecalEnum.Property are
-                //
+            // parameterized COM properties that the decompiler flattened to
-                // For now, register the known built-in filters that ship with Decal.
+            // simple getters. They need to be called via IDispatch to pass
-                // When Decal.Core is implemented, this will read from live config.
+            // the category name / property name arguments.
-            }
+            //
-            catch
+            // Filters are registered externally (by plugins/installer in the
-            {
+            // registry under HKLM\SOFTWARE\Decal\FileFilters\{CLSID}) and
-                // Config may not be available yet
+            // will be loaded once Decal.Core exposes the Configuration enum.
-            }
+            //
+            // DecalDat still works without filters — callers get raw IDatStream
+            // objects. Filters are an optional layer that parses the stream into
+            // typed objects (e.g., spell tables, character data).
         }
 
         public void BeforePlugins()

Managed/Decal.DecalDat/DatStreamImpl.cs

@@ -38,8 +38,14 @@ namespace Decal.DecalDat
             var buf = new byte[Bytes];
             _file.Read(buf, 0, Bytes);
 
-            // Copy to unmanaged buffer starting at ref Buffer
+            // Pin the destination buffer for the duration of the copy
-            Marshal.Copy(buf, 0, GetBufferPtr(ref Buffer), Bytes);
+            unsafe
+            {
+                fixed (byte* p = &Buffer)
+                {
+                    Marshal.Copy(buf, 0, new IntPtr(p), Bytes);
+                }
+            }
         }
 
         public string Read(int Bytes)
@@ -49,22 +55,13 @@ namespace Decal.DecalDat
             var buf = new byte[Bytes];
             int read = _file.Read(buf, 0, Bytes);
 
-            // Return as binary BSTR (same as SysAllocStringByteLen in C++)
+            // Match C++ SysAllocStringByteLen: pack raw bytes into WCHAR pairs.
-            unsafe
+            // When the CLR marshals this string to BSTR, the raw bytes are preserved
-            {
+            // byte-for-byte (each char = 2 raw bytes, little-endian).
-                fixed (byte* p = buf)
+            int charCount = (read + 1) / 2;
-                {
+            var chars = new char[charCount];
-                    return Marshal.PtrToStringAnsi(new IntPtr(p), read);
+            Buffer.BlockCopy(buf, 0, chars, 0, read);
-                }
+            return new string(chars, 0, charCount);
-            }
-        }
-
-        private static unsafe IntPtr GetBufferPtr(ref byte buffer)
-        {
-            fixed (byte* p = &buffer)
-            {
-                return new IntPtr(p);
-            }
         }
     }
 }