using System.Numerics;
using System.Diagnostics.CodeAnalysis;
using AcDream.Content;
using AcDream.Content.Vfx;
using AcDream.Core.Vfx;
using DatReaderWriter;
using DatReaderWriter.DBObjs;
using DatReaderWriter.Lib.IO;
using DatReaderWriter.Options;
using DatReaderWriter.Types;
using DatAnimation = DatReaderWriter.DBObjs.Animation;
using DatPhysicsScript = DatReaderWriter.DBObjs.PhysicsScript;
namespace AcDream.Content.Tests.Vfx;
public sealed class RetailDatLoaderTests
{
///
/// Runs both callers on dedicated threads rather than thread-pool work
/// items.
///
///
/// These tests measure whether the LOADER serialises reads, and the
/// interesting ones assert on MaxConcurrentReads. A pair of
/// work items does not guarantee two
/// workers are ever in flight at once: on a saturated or low-core pool -
/// four cores on a hosted CI runner - the second item simply queues behind
/// the first, both 40 ms reads run back to back, and the assertion fails
/// for a reason that has nothing to do with the loader. Observed exactly
/// that way in CI on both operating systems, and reproduced locally by
/// pinning the suite to two CPUs (5 failures in 6).
/// on the default
/// scheduler asks for a thread per callback, which makes the concurrency
/// the assertions measure actually available. The assertions themselves are
/// unchanged and still fail if the loader serialises - and the coalescing
/// cases get stronger, because two callers genuinely in flight is the
/// situation coalescing exists for, where a sequential pair only ever
/// exercised a cache hit.
///
private static Task InParallel(Func first, Func second) =>
Task.WhenAll(
Task.Factory.StartNew(
first,
CancellationToken.None,
TaskCreationOptions.LongRunning,
TaskScheduler.Default),
Task.Factory.StartNew(
second,
CancellationToken.None,
TaskCreationOptions.LongRunning,
TaskScheduler.Default));
private sealed class RawDatabase : IDatDatabase
{
private readonly Dictionary _entries = new();
private int _activeReads;
private int _maxConcurrentReads;
private int _totalReads;
private Barrier? _concurrencyGate;
public DatDatabase Db => null!;
public int Iteration => 0;
public int ReadDelayMilliseconds { get; set; }
public int MaxConcurrentReads => Volatile.Read(ref _maxConcurrentReads);
public int TotalReads => Volatile.Read(ref _totalReads);
public void Add(uint id, byte[] bytes) => _entries[id] = bytes;
///
/// Forces exactly concurrent
/// callers to
/// rendezvous before any of them is allowed to return, proving true
/// overlap deterministically. Earlier versions of these tests proved
/// overlap by racing a fixed
/// sleep against thread-pool scheduling latency; under system-wide
/// CPU contention (e.g. `dotnet test` running many test projects
/// concurrently) that race could lose, producing an intermittent
/// failure with no code defect (issue #248). A barrier removes the
/// wall-clock dependency entirely — the assertion holds regardless
/// of how slow scheduling is, as long as both callers eventually run.
///
public void ArmConcurrencyGate(int participantCount) =>
_concurrencyGate = new Barrier(participantCount);
public IEnumerable GetAllIdsOfType() where T : IDBObj => _entries.Keys;
public bool TryGet(uint fileId, [MaybeNullWhen(false)] out T value) where T : IDBObj
{
value = default;
return false;
}
public bool TryGetFileBytes(uint fileId, [MaybeNullWhen(false)] out byte[] value)
{
Interlocked.Increment(ref _totalReads);
int active = Interlocked.Increment(ref _activeReads);
int observed;
do
{
observed = Volatile.Read(ref _maxConcurrentReads);
}
while (active > observed
&& Interlocked.CompareExchange(ref _maxConcurrentReads, active, observed) != observed);
try
{
_concurrencyGate?.SignalAndWait();
if (ReadDelayMilliseconds > 0)
Thread.Sleep(ReadDelayMilliseconds);
return _entries.TryGetValue(fileId, out value);
}
finally
{
Interlocked.Decrement(ref _activeReads);
}
}
public bool TryGetFileBytes(uint fileId, ref byte[] bytes, out int bytesRead)
{
if (!_entries.TryGetValue(fileId, out byte[]? value))
{
bytesRead = 0;
return false;
}
if (bytes.Length < value.Length)
bytes = new byte[value.Length];
value.CopyTo(bytes, 0);
bytesRead = value.Length;
return true;
}
public bool TrySave(T obj, int iteration = 0) where T : IDBObj => false;
public void Dispose() { }
}
[Fact]
public void PhysicsScript_BlockingHookRetainsPayloadAndFollowingCursor()
{
DatPhysicsScript script = RetailPhysicsScriptLoader.Parse(
ProjectileVfxDatFixtures.PhysicsScriptCreateBlockingThenAnimationDone);
Assert.Equal(0x3300F001u, script.Id);
Assert.Equal(2, script.ScriptData.Count);
Assert.Equal(0.0, script.ScriptData[0].StartTime);
var blocking = Assert.IsType(
script.ScriptData[0].Hook);
Assert.Equal(0x3200F001u, blocking.EmitterInfoId.DataId);
Assert.Equal(0xFFFFFFFFu, blocking.PartIndex);
Assert.Equal(new Vector3(1f, 2f, 3f), blocking.Offset.Origin);
Assert.Equal(new Quaternion(0.5f, -0.5f, -0.5f, 0.5f),
blocking.Offset.Orientation);
Assert.Equal(7u, blocking.EmitterId);
Assert.Equal(1.25, script.ScriptData[1].StartTime);
Assert.IsType(script.ScriptData[1].Hook);
}
[Fact]
public void Animation_BlockingHookRetainsPayloadAndFollowingCursor()
{
DatAnimation animation = RetailAnimationLoader.Parse(
ProjectileVfxDatFixtures.AnimationCreateBlockingThenAnimationDone);
Assert.Equal(0x0300F001u, animation.Id);
AnimationFrame frame = Assert.Single(animation.PartFrames);
Assert.Equal(2, frame.Hooks.Count);
var blocking = Assert.IsType(frame.Hooks[0]);
Assert.Equal(0x3200F002u, blocking.EmitterInfoId.DataId);
Assert.Equal(0u, blocking.PartIndex);
Assert.Equal(new Vector3(-1f, 4f, 0.25f), blocking.Offset.Origin);
Assert.Equal(Quaternion.Identity, blocking.Offset.Orientation);
Assert.Equal(9u, blocking.EmitterId);
Assert.IsType(frame.Hooks[1]);
}
[Fact]
public void OrdinaryPhysicsScript_MatchesPackageDecoder()
{
byte[] bytes = ProjectileVfxDatFixtures.OrdinaryPhysicsScript;
DatPhysicsScript retail = RetailPhysicsScriptLoader.Parse(bytes);
var package = new DatPhysicsScript();
var reader = new DatBinReader(bytes);
Assert.True(package.Unpack(reader));
Assert.Equal(reader.Length, reader.Offset);
Assert.Equal(package.Id, retail.Id);
Assert.Equal(package.ScriptData.Count, retail.ScriptData.Count);
Assert.Equal(package.ScriptData[0].StartTime, retail.ScriptData[0].StartTime);
Assert.Equal(package.ScriptData[0].Hook.HookType, retail.ScriptData[0].Hook.HookType);
Assert.Equal(package.ScriptData[0].Hook.Direction, retail.ScriptData[0].Hook.Direction);
}
[Fact]
public void OrdinaryAnimation_MatchesPackageDecoder()
{
byte[] bytes = ProjectileVfxDatFixtures.OrdinaryAnimation;
DatAnimation retail = RetailAnimationLoader.Parse(bytes);
var package = new DatAnimation();
var reader = new DatBinReader(bytes);
Assert.True(package.Unpack(reader));
Assert.Equal(reader.Length, reader.Offset);
Assert.Equal(package.Id, retail.Id);
Assert.Equal(package.Flags, retail.Flags);
Assert.Equal(package.NumParts, retail.NumParts);
Assert.Equal(package.PartFrames.Count, retail.PartFrames.Count);
Assert.Equal(package.PartFrames[0].Hooks[0].HookType,
retail.PartFrames[0].Hooks[0].HookType);
Assert.Equal(package.PartFrames[0].Hooks[0].Direction,
retail.PartFrames[0].Hooks[0].Direction);
}
[Fact]
public void PhysicsScript_SortsHooksByRetailStartTimeAfterDecode()
{
byte[] bytes = ProjectileVfxDatFixtures.PhysicsScriptCreateBlockingThenAnimationDone.ToArray();
// The fixture stores starts 0.0 then 1.25. Make the first later than
// the second without changing either hook payload or cursor.
System.Buffers.Binary.BinaryPrimitives.WriteDoubleLittleEndian(
bytes.AsSpan(8), 2.0);
DatPhysicsScript script = RetailPhysicsScriptLoader.Parse(bytes);
Assert.Equal(1.25, script.ScriptData[0].StartTime);
Assert.IsType(script.ScriptData[0].Hook);
Assert.Equal(2.0, script.ScriptData[1].StartTime);
Assert.IsType(script.ScriptData[1].Hook);
}
[Fact]
public void AnimationDidValidation_UsesRetailHighByte()
{
Assert.True(RetailAnimationLoader.IsAnimationDid(0x03010000u));
Assert.True(RetailAnimationLoader.IsAnimationDid(0x03FFFFFFu));
Assert.False(RetailAnimationLoader.IsAnimationDid(0x04010000u));
}
[Fact]
public void Loaders_AcceptHighIndexesCacheIdentityAndRejectEmbeddedIdMismatch()
{
const uint scriptDid = 0x33010000u;
const uint animationDid = 0x03010000u;
var portal = new RawDatabase();
byte[] scriptBytes = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
byte[] animationBytes = ProjectileVfxDatFixtures.OrdinaryAnimation.ToArray();
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(scriptBytes, scriptDid);
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(animationBytes, animationDid);
portal.Add(scriptDid, scriptBytes);
portal.Add(animationDid, animationBytes);
var scripts = new RetailPhysicsScriptLoader(portal);
var animations = new RetailAnimationLoader(portal);
DatPhysicsScript script = Assert.IsType(scripts.LoadPhysicsScript(scriptDid));
DatAnimation animation = Assert.IsType(animations.LoadAnimation(animationDid));
Assert.Same(script, scripts.LoadPhysicsScript(scriptDid));
Assert.Same(animation, animations.LoadAnimation(animationDid));
const uint wrongScriptKey = 0x33010001u;
const uint wrongAnimationKey = 0x03010001u;
portal.Add(wrongScriptKey, scriptBytes);
portal.Add(wrongAnimationKey, animationBytes);
Assert.Throws(() => scripts.LoadPhysicsScript(wrongScriptKey));
Assert.Throws(() => animations.LoadAnimation(wrongAnimationKey));
}
[Fact]
public void AnimationCache_CountPressureEvictsLruWithoutInvalidatingLiveReferences()
{
const uint firstDid = 0x03010020u;
const uint secondDid = 0x03010021u;
const uint thirdDid = 0x03010022u;
var portal = new RawDatabase();
portal.Add(firstDid, AnimationBytes(firstDid));
portal.Add(secondDid, AnimationBytes(secondDid));
portal.Add(thirdDid, AnimationBytes(thirdDid));
var loader = new RetailAnimationLoader(
portal,
maximumEstimatedBytes: long.MaxValue,
maximumEntries: 2);
DatAnimation first = Assert.IsType(
loader.LoadAnimation(firstDid));
_ = Assert.IsType(loader.LoadAnimation(secondDid));
_ = Assert.IsType(loader.LoadAnimation(thirdDid));
AnimationCacheDiagnostics pressured = loader.Diagnostics;
Assert.Equal(2, pressured.Count);
Assert.Equal(1, pressured.Stats.Evictions);
Assert.Equal(firstDid, first.Id);
DatAnimation reloaded = Assert.IsType(
loader.LoadAnimation(firstDid));
Assert.NotSame(first, reloaded);
Assert.Equal(4, portal.TotalReads);
Assert.Equal(2, loader.Diagnostics.Stats.Evictions);
}
[Fact]
public void AnimationCache_OversizeEntryIsServedButNeverRetained()
{
const uint animationDid = 0x03010023u;
var portal = new RawDatabase();
portal.Add(animationDid, AnimationBytes(animationDid));
var loader = new RetailAnimationLoader(
portal,
maximumEstimatedBytes: 1,
maximumEntries: 2);
Assert.NotNull(loader.LoadAnimation(animationDid));
Assert.NotNull(loader.LoadAnimation(animationDid));
Assert.Equal(0, loader.Diagnostics.Count);
Assert.Equal(0, loader.Diagnostics.EstimatedBytes);
Assert.Equal(2, portal.TotalReads);
}
[Fact]
public async Task AnimationCache_CoalescesSameDidAndAllowsUnrelatedReadsInParallel()
{
const uint firstDid = 0x03010024u;
const uint secondDid = 0x03010025u;
var portal = new RawDatabase();
portal.Add(firstDid, AnimationBytes(firstDid));
portal.Add(secondDid, AnimationBytes(secondDid));
var loader = new RetailAnimationLoader(portal);
DatAnimation?[] firstPair = await InParallel(
() => loader.LoadAnimation(firstDid),
() => loader.LoadAnimation(firstDid));
Assert.Same(firstPair[0], firstPair[1]);
Assert.Equal(1, portal.TotalReads);
// secondDid and 0x03010026u are unrelated keys, so the loader must
// not serialize their reads against each other. Prove that with a
// rendezvous gate rather than a sleep race (see ArmConcurrencyGate).
portal.ArmConcurrencyGate(2);
await Task.WhenAll(
Task.Run(() => loader.LoadAnimation(secondDid)),
Task.Run(() => loader.LoadAnimation(0x03010026u)));
Assert.Equal(2, portal.MaxConcurrentReads);
Assert.Equal(3, portal.TotalReads);
}
[Fact]
public async Task PhysicsScriptLoader_AllowsConcurrentFirstReads()
{
const uint firstDid = 0x33010010u;
const uint secondDid = 0x33010011u;
var portal = new RawDatabase();
byte[] first = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
byte[] second = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(first, firstDid);
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(second, secondDid);
portal.Add(firstDid, first);
portal.Add(secondDid, second);
// firstDid and secondDid are unrelated keys; prove the loader runs
// their reads concurrently with a rendezvous gate rather than a
// sleep race (see ArmConcurrencyGate).
portal.ArmConcurrencyGate(2);
var loader = new RetailPhysicsScriptLoader(portal);
await InParallel(
() => loader.LoadPhysicsScript(firstDid),
() => loader.LoadPhysicsScript(secondDid));
Assert.Equal(2, portal.MaxConcurrentReads);
Assert.Equal(2, portal.TotalReads);
}
private static byte[] AnimationBytes(uint id)
{
byte[] bytes =
ProjectileVfxDatFixtures.OrdinaryAnimation.ToArray();
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(
bytes,
id);
return bytes;
}
[Fact]
public async Task PhysicsScriptLoader_CoalescesConcurrentReadsForTheSameDid()
{
const uint scriptDid = 0x33010012u;
var portal = new RawDatabase { ReadDelayMilliseconds = 40 };
byte[] bytes = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(bytes, scriptDid);
portal.Add(scriptDid, bytes);
var loader = new RetailPhysicsScriptLoader(portal);
DatPhysicsScript?[] loaded = await InParallel(
() => loader.LoadPhysicsScript(scriptDid),
() => loader.LoadPhysicsScript(scriptDid));
Assert.All(loaded, Assert.NotNull);
Assert.Same(loaded[0], loaded[1]);
Assert.Equal(1, portal.TotalReads);
}
[Fact]
public void Parsers_RejectTrailingOrTruncatedEntries()
{
byte[] script = ProjectileVfxDatFixtures.PhysicsScriptCreateBlockingThenAnimationDone;
byte[] animation = ProjectileVfxDatFixtures.AnimationCreateBlockingThenAnimationDone;
Assert.ThrowsAny(() =>
RetailPhysicsScriptLoader.Parse(script.AsMemory(0, script.Length - 1)));
Assert.ThrowsAny(() =>
RetailAnimationLoader.Parse(animation.AsMemory(0, animation.Length - 1)));
Assert.Throws(() =>
RetailPhysicsScriptLoader.Parse(script.Concat(new byte[] { 0 }).ToArray()));
Assert.Throws(() =>
RetailAnimationLoader.Parse(animation.Concat(new byte[] { 0 }).ToArray()));
}
[Fact]
public void PhysicsScriptParserRejectsNonFiniteHookTime()
{
byte[] script = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
System.Buffers.Binary.BinaryPrimitives.WriteInt64LittleEndian(
script.AsSpan(8, 8),
BitConverter.DoubleToInt64Bits(double.NaN));
Assert.Throws(() => RetailPhysicsScriptLoader.Parse(script));
}
[Fact]
public void Parsers_RejectUnsupportedHookTypesWithoutReturningPartialObjects()
{
byte[] script = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
byte[] animation = ProjectileVfxDatFixtures.OrdinaryAnimation.ToArray();
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(
script.AsSpan(16), 0xDEADBEEFu);
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(
animation.AsSpan(20), 0xDEADBEEFu);
Assert.ThrowsAny(() => RetailPhysicsScriptLoader.Parse(script));
Assert.ThrowsAny(() => RetailAnimationLoader.Parse(animation));
}
[Fact]
public void InstalledBlockingScripts_ParseThroughRetailShape_WhenDatsAvailable()
{
string? datDir = ResolveDatDirectory();
if (datDir is null)
return;
using var dats = new DatCollection(datDir, DatAccessType.Read);
using var boundedDats = new DatCollectionAdapter(dats);
var loader = new RetailPhysicsScriptLoader(boundedDats);
foreach (uint id in new[] { 0x33000AEAu, 0x33000AF8u })
{
DatPhysicsScript script = Assert.IsType(
loader.LoadPhysicsScript(id));
Assert.Contains(script.ScriptData,
item => item.Hook is RetailCreateBlockingParticleHook);
}
}
private static string? ResolveDatDirectory()
{
string? configured = System.Environment.GetEnvironmentVariable("ACDREAM_DAT_DIR");
if (!string.IsNullOrWhiteSpace(configured)
&& File.Exists(Path.Combine(configured, "client_portal.dat")))
{
return configured;
}
const string installed = @"C:\Turbine\Asheron's Call";
return File.Exists(Path.Combine(installed, "client_portal.dat"))
? installed
: null;
}
}