Fix DecalDat to work with real AC DAT files
The DAT file reader had several bugs inherited from the old C++ reference code, which targeted an older format version. Verified and fixed against real client_portal.dat and client_cell_1.dat files: - Fix header offset: BTree root is at 0x160, not 0x148 (file size field) - Fix BTree entry size: 24 bytes (flags+id+offset+size+timestamp), not 12 - Fix sector-chain node reading: BTree nodes span multiple sectors via linked-list headers; must assemble node data across sector boundaries - Fix DatStreamImpl.Read() BSTR handling: use Buffer.BlockCopy to match C++ SysAllocStringByteLen instead of Marshal.PtrToStringAnsi - Fix DatStreamImpl.ReadBinary() pointer lifetime: inline fixed block to keep destination buffer pinned during Marshal.Copy - Document LoadFilters() dependency on parameterized COM properties in IDecalCore.Configuration that need IDispatch to call correctly Add smoke test project (13/13 tests pass against real DAT files). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
erik
parent
c0d1135431
commit
f0b6fedc9b
5 changed files with 356 additions and 56 deletions
|
|
@ -1,8 +1,11 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
using System.IO.MemoryMappedFiles;
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("Decal.DecalDat.Tests")]
|
||||
|
||||
namespace Decal.DecalDat
|
||||
{
|
||||
/// <summary>
|
||||
|
|
@ -11,17 +14,38 @@ namespace Decal.DecalDat
|
|||
/// </summary>
|
||||
internal sealed class DatFile : IDisposable
|
||||
{
|
||||
private const int FileCount = 62;
|
||||
private const int FileCountOffset = 0x03E;
|
||||
private const int RootDirPtrOffset = 0x148;
|
||||
// DAT header layout (starting at 0x140):
|
||||
// 0x140: "BT" magic (0x5442)
|
||||
// 0x144: block/sector size
|
||||
// 0x148: file size
|
||||
// 0x14C: data set type (1=portal, 2=cell)
|
||||
// 0x150: data subset
|
||||
// 0x154: free head
|
||||
// 0x158: free tail
|
||||
// 0x15C: free count
|
||||
// 0x160: BTree root directory offset
|
||||
private const int RootDirPtrOffset = 0x160;
|
||||
|
||||
// Directory entry layout (packed):
|
||||
// DWORD subdirs[62] = 248 bytes
|
||||
// DWORD fileCount = 4 bytes
|
||||
// FileEntry files[62] = 744 bytes (62 * 12)
|
||||
// Total = 996 bytes
|
||||
private const int SubdirsSize = FileCount * 4;
|
||||
private const int FileEntrySize = 12; // 3 DWORDs: ID, Offset, Size
|
||||
// BTree node data layout (within sector chain, after skipping sector headers):
|
||||
// DWORD branches[62] = 248 bytes (child pointers; 0 = leaf)
|
||||
// DWORD entryCount = 4 bytes
|
||||
// FileEntry entries[62] = 1488 bytes (62 * 24)
|
||||
// Total node data = 1740 bytes
|
||||
//
|
||||
// Nodes are stored in sector chains: each sector's first 4 bytes = next sector
|
||||
// offset, with usable data in bytes [4..sectorSize-1].
|
||||
//
|
||||
// Each FileEntry (24 bytes):
|
||||
// DWORD flags (+0)
|
||||
// DWORD objectId (+4) — the file ID used for lookup
|
||||
// DWORD fileOffset (+8) — sector offset to file data
|
||||
// DWORD fileSize (+12) — size in bytes
|
||||
// UINT64 timestamp (+16) — Windows FILETIME
|
||||
private const int MaxEntries = 62;
|
||||
private const int BranchCount = MaxEntries; // 62 child pointers
|
||||
private const int BranchesSize = BranchCount * 4; // 248
|
||||
private const int FileEntrySize = 24; // flags(4) + id(4) + offset(4) + size(4) + timestamp(8)
|
||||
private const int MaxNodeDataSize = BranchesSize + 4 + MaxEntries * FileEntrySize; // 1740
|
||||
|
||||
private MemoryMappedFile _mmf;
|
||||
private MemoryMappedViewAccessor _accessor;
|
||||
|
|
@ -44,30 +68,76 @@ namespace Decal.DecalDat
|
|||
return FindFile(rootDirOffset, fileId);
|
||||
}
|
||||
|
||||
private DatFileEntry FindFile(uint dirOffset, uint fileId)
|
||||
/// <summary>
|
||||
/// Reads BTree node data from a sector chain into a contiguous byte array.
|
||||
/// Each sector's first 4 bytes is the next-sector pointer; data follows.
|
||||
/// </summary>
|
||||
private byte[] ReadNodeData(uint sectorOffset)
|
||||
{
|
||||
// Read file count from directory
|
||||
uint fileCount = _accessor.ReadUInt32(dirOffset + SubdirsSize);
|
||||
var data = new byte[MaxNodeDataSize];
|
||||
int dataPerSector = _sectorSize - 4;
|
||||
int copied = 0;
|
||||
uint currentSector = sectorOffset;
|
||||
|
||||
while (copied < MaxNodeDataSize && currentSector != 0 && currentSector < (uint)_fileLength)
|
||||
{
|
||||
int toCopy = Math.Min(dataPerSector, MaxNodeDataSize - copied);
|
||||
// Bounds check: ensure we don't read past end of file
|
||||
long readStart = currentSector + 4;
|
||||
if (readStart + toCopy > _fileLength)
|
||||
toCopy = (int)(_fileLength - readStart);
|
||||
if (toCopy <= 0) break;
|
||||
|
||||
_accessor.ReadArray(readStart, data, copied, toCopy);
|
||||
copied += toCopy;
|
||||
|
||||
if (copied < MaxNodeDataSize)
|
||||
currentSector = _accessor.ReadUInt32(currentSector);
|
||||
else
|
||||
break;
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private static uint ReadUInt32(byte[] data, int offset)
|
||||
{
|
||||
return BitConverter.ToUInt32(data, offset);
|
||||
}
|
||||
|
||||
private DatFileEntry FindFile(uint sectorOffset, uint fileId)
|
||||
{
|
||||
if (sectorOffset >= (uint)_fileLength)
|
||||
throw new FileNotFoundException($"BTree branch 0x{sectorOffset:X8} is out of bounds (file=0x{_fileLength:X})");
|
||||
|
||||
byte[] node = ReadNodeData(sectorOffset);
|
||||
|
||||
// Parse node: branches[62] (248 bytes) + entryCount (4 bytes) + entries
|
||||
uint entryCount = ReadUInt32(node, BranchesSize);
|
||||
|
||||
if (entryCount > MaxEntries)
|
||||
throw new InvalidDataException($"BTree node at 0x{sectorOffset:X8} has invalid entry count {entryCount}");
|
||||
|
||||
// Binary search through sorted file entries
|
||||
int lo = 0, hi = (int)fileCount - 1;
|
||||
int lo = 0, hi = (int)entryCount - 1;
|
||||
|
||||
while (lo <= hi)
|
||||
{
|
||||
int mid = (lo + hi) / 2;
|
||||
long entryOffset = dirOffset + SubdirsSize + 4 + (mid * FileEntrySize);
|
||||
uint entryId = _accessor.ReadUInt32(entryOffset);
|
||||
int entryOffset = BranchesSize + 4 + (mid * FileEntrySize);
|
||||
// Entry layout: flags(+0), objectId(+4), fileOffset(+8), fileSize(+12), timestamp(+16)
|
||||
uint entryId = ReadUInt32(node, entryOffset + 4);
|
||||
|
||||
if (entryId == fileId)
|
||||
{
|
||||
uint offset = _accessor.ReadUInt32(entryOffset + 4);
|
||||
uint size = _accessor.ReadUInt32(entryOffset + 8);
|
||||
uint offset = ReadUInt32(node, entryOffset + 8);
|
||||
uint size = ReadUInt32(node, entryOffset + 12);
|
||||
return new DatFileEntry(this, offset, size);
|
||||
}
|
||||
else if (fileId < entryId)
|
||||
{
|
||||
// Recurse into left subdirectory
|
||||
uint subdir = _accessor.ReadUInt32(dirOffset + (uint)(mid * 4));
|
||||
// Recurse into left child branch
|
||||
uint subdir = ReadUInt32(node, mid * 4);
|
||||
if (subdir != 0)
|
||||
{
|
||||
try
|
||||
|
|
@ -87,8 +157,8 @@ namespace Decal.DecalDat
|
|||
}
|
||||
}
|
||||
|
||||
// Check rightmost subdirectory
|
||||
uint rightSubdir = _accessor.ReadUInt32(dirOffset + (uint)(lo * 4));
|
||||
// Check rightmost child branch
|
||||
uint rightSubdir = ReadUInt32(node, lo * 4);
|
||||
if (rightSubdir != 0)
|
||||
{
|
||||
return FindFile(rightSubdir, fileId);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue