openDecal/Managed/Decal.DecalDat/DatFile.cs

using System;
using System.IO;
using System.IO.MemoryMappedFiles;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;

[assembly: InternalsVisibleTo("Decal.DecalDat.Tests")]

namespace Decal.DecalDat
{
    /// <summary>
    /// Low-level reader for Asheron's Call DAT archive files.
    /// DAT files use a sector-based linked-list format with a hierarchical directory.
    /// </summary>
    internal sealed class DatFile : IDisposable
    {
        // DAT header layout (starting at 0x140):
        //   0x140: "BT" magic (0x5442)
        //   0x144: block/sector size
        //   0x148: file size
        //   0x14C: data set type (1=portal, 2=cell)
        //   0x150: data subset
        //   0x154: free head
        //   0x158: free tail
        //   0x15C: free count
        //   0x160: BTree root directory offset
        private const int RootDirPtrOffset = 0x160;

        // BTree node data layout (within sector chain, after skipping sector headers):
        //   DWORD branches[62]           = 248 bytes  (child pointers; 0 = leaf)
        //   DWORD entryCount             =   4 bytes
        //   FileEntry entries[62]        = 1488 bytes (62 * 24)
        //   Total node data              = 1740 bytes
        //
        // Nodes are stored in sector chains: each sector's first 4 bytes = next sector
        // offset, with usable data in bytes [4..sectorSize-1].
        //
        // Each FileEntry (24 bytes):
        //   DWORD flags      (+0)
        //   DWORD objectId   (+4)   — the file ID used for lookup
        //   DWORD fileOffset (+8)   — sector offset to file data
        //   DWORD fileSize   (+12)  — size in bytes
        //   UINT64 timestamp (+16)  — Windows FILETIME
        private const int MaxEntries = 62;
        private const int BranchCount = MaxEntries; // 62 child pointers
        private const int BranchesSize = BranchCount * 4; // 248
        private const int FileEntrySize = 24; // flags(4) + id(4) + offset(4) + size(4) + timestamp(8)
        private const int MaxNodeDataSize = BranchesSize + 4 + MaxEntries * FileEntrySize; // 1740

        private MemoryMappedFile _mmf;
        private MemoryMappedViewAccessor _accessor;
        private readonly int _sectorSize;
        private readonly long _fileLength;

        public DatFile(string filename, int sectorSize = 256)
        {
            _sectorSize = sectorSize;

            var fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
            _fileLength = fs.Length;
            _mmf = MemoryMappedFile.CreateFromFile(fs, null, 0, MemoryMappedFileAccess.Read, HandleInheritability.None, false);
            _accessor = _mmf.CreateViewAccessor(0, _fileLength, MemoryMappedFileAccess.Read);
        }

        public DatFileEntry GetFile(uint fileId)
        {
            uint rootDirOffset = _accessor.ReadUInt32(RootDirPtrOffset);
            return FindFile(rootDirOffset, fileId);
        }

        /// <summary>
        /// Reads BTree node data from a sector chain into a contiguous byte array.
        /// Each sector's first 4 bytes is the next-sector pointer; data follows.
        /// </summary>
        private byte[] ReadNodeData(uint sectorOffset)
        {
            var data = new byte[MaxNodeDataSize];
            int dataPerSector = _sectorSize - 4;
            int copied = 0;
            uint currentSector = sectorOffset;

            while (copied < MaxNodeDataSize && currentSector != 0 && currentSector < (uint)_fileLength)
            {
                int toCopy = Math.Min(dataPerSector, MaxNodeDataSize - copied);
                // Bounds check: ensure we don't read past end of file
                long readStart = currentSector + 4;
                if (readStart + toCopy > _fileLength)
                    toCopy = (int)(_fileLength - readStart);
                if (toCopy <= 0) break;

                _accessor.ReadArray(readStart, data, copied, toCopy);
                copied += toCopy;

                if (copied < MaxNodeDataSize)
                    currentSector = _accessor.ReadUInt32(currentSector);
                else
                    break;
            }

            return data;
        }

        private static uint ReadUInt32(byte[] data, int offset)
        {
            return BitConverter.ToUInt32(data, offset);
        }

        private DatFileEntry FindFile(uint sectorOffset, uint fileId)
        {
            if (sectorOffset >= (uint)_fileLength)
                throw new FileNotFoundException($"BTree branch 0x{sectorOffset:X8} is out of bounds (file=0x{_fileLength:X})");

            byte[] node = ReadNodeData(sectorOffset);

            // Parse node: branches[62] (248 bytes) + entryCount (4 bytes) + entries
            uint entryCount = ReadUInt32(node, BranchesSize);

            if (entryCount > MaxEntries)
                throw new InvalidDataException($"BTree node at 0x{sectorOffset:X8} has invalid entry count {entryCount}");

            // Binary search through sorted file entries
            int lo = 0, hi = (int)entryCount - 1;

            while (lo <= hi)
            {
                int mid = (lo + hi) / 2;
                int entryOffset = BranchesSize + 4 + (mid * FileEntrySize);
                // Entry layout: flags(+0), objectId(+4), fileOffset(+8), fileSize(+12), timestamp(+16)
                uint entryId = ReadUInt32(node, entryOffset + 4);

                if (entryId == fileId)
                {
                    uint offset = ReadUInt32(node, entryOffset + 8);
                    uint size = ReadUInt32(node, entryOffset + 12);
                    return new DatFileEntry(this, offset, size);
                }
                else if (fileId < entryId)
                {
                    // Recurse into left child branch
                    uint subdir = ReadUInt32(node, mid * 4);
                    if (subdir != 0)
                    {
                        try
                        {
                            return FindFile(subdir, fileId);
                        }
                        catch (FileNotFoundException)
                        {
                            // Not in this subtree
                        }
                    }
                    hi = mid - 1;
                }
                else
                {
                    lo = mid + 1;
                }
            }

            // Check rightmost child branch
            uint rightSubdir = ReadUInt32(node, lo * 4);
            if (rightSubdir != 0)
            {
                return FindFile(rightSubdir, fileId);
            }

            throw new FileNotFoundException($"File 0x{fileId:X8} not found in DAT");
        }

        internal int SectorSize => _sectorSize;
        internal int DataPerSector => _sectorSize - 4; // First 4 bytes = next sector pointer

        internal void ReadBytes(long offset, byte[] buffer, int bufferOffset, int count)
        {
            _accessor.ReadArray(offset, buffer, bufferOffset, count);
        }

        internal uint ReadUInt32(long offset)
        {
            return _accessor.ReadUInt32(offset);
        }

        public void Dispose()
        {
            _accessor?.Dispose();
            _accessor = null;
            _mmf?.Dispose();
            _mmf = null;
        }
    }

    /// <summary>
    /// Represents a single file within a DAT archive.
    /// Reads through a linked list of sectors.
    /// </summary>
    internal sealed class DatFileEntry
    {
        private readonly DatFile _source;
        private readonly uint _firstSectorOffset;
        private readonly uint _size;

        private uint _currentSectorOffset;
        private int _posInSector; // position within current sector's data area
        private int _totalRead;  // total bytes read so far

        public DatFileEntry(DatFile source, uint sectorOffset, uint size)
        {
            _source = source;
            _firstSectorOffset = sectorOffset;
            _size = size;
            Reset();
        }

        public int Size => (int)_size;
        public int Tell => _totalRead;

        public void Reset()
        {
            _currentSectorOffset = _firstSectorOffset;
            _posInSector = 0;
            _totalRead = 0;
        }

        public void Skip(int bytes)
        {
            int remaining = bytes;
            while (remaining > 0)
            {
                int dataPerSector = _source.DataPerSector;
                int availableInSector = dataPerSector - _posInSector;

                if (remaining < availableInSector)
                {
                    _posInSector += remaining;
                    _totalRead += remaining;
                    return;
                }

                // Skip rest of this sector and move to next
                _totalRead += availableInSector;
                remaining -= availableInSector;
                MoveToNextSector();
            }
        }

        public int Read(byte[] buffer, int offset, int count)
        {
            int remaining = Math.Min(count, (int)_size - _totalRead);
            int totalCopied = 0;

            while (remaining > 0)
            {
                int dataPerSector = _source.DataPerSector;
                int availableInSector = dataPerSector - _posInSector;
                int toCopy = Math.Min(remaining, availableInSector);

                // Data starts at sector offset + 4 (skip next-sector pointer)
                long readOffset = _currentSectorOffset + 4 + _posInSector;
                _source.ReadBytes(readOffset, buffer, offset + totalCopied, toCopy);

                _posInSector += toCopy;
                _totalRead += toCopy;
                totalCopied += toCopy;
                remaining -= toCopy;

                if (_posInSector >= dataPerSector && remaining > 0)
                {
                    MoveToNextSector();
                }
            }

            return totalCopied;
        }

        private void MoveToNextSector()
        {
            // Next sector pointer is stored in first 4 bytes of current sector
            uint nextSector = _source.ReadUInt32(_currentSectorOffset);
            _currentSectorOffset = nextSector;
            _posInSector = 0;
        }
    }
}