c7861020e1
Fixing the two failures that were stopping portable-headless earlier let the job reach AcDream.Content.Tests for the first time on either operating system - the test loop exits on the first failing project, so the windows leg had never got past the apt step and the ubuntu leg had never got past Core.Net. Two RetailDatLoaderTests cases were waiting there, and they failed on both. Both assert on RawDatabase.MaxConcurrentReads after issuing two Task.Run reads that each block 40 ms in Thread.Sleep. A pair of pool work items is not a guarantee of two workers in flight: on a low-core or saturated pool the second queues behind the first, the reads run back to back, MaxConcurrentReads stays 1, and the assertion fails for a reason that has nothing to do with the loader. Pinning the suite to two CPUs on Ubuntu reproduces it 5 times in 6; Windows is clean 6 of 6 at sixteen cores, which is why nobody had seen it. The pairs now start with TaskCreationOptions.LongRunning on the default scheduler, which asks for a thread each. No assertion is changed - they still fail if the loader serialises. The two coalescing cases moved onto the same helper on purpose: two callers genuinely in flight is the situation coalescing exists for, and a sequential pair was only ever exercising a cache hit. Ten of ten clean under the same pin. Release build green. App tests 4,152 / 3 skipped. Content 124 / 124. Filed as #255. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
441 lines
18 KiB
C#
441 lines
18 KiB
C#
using System.Numerics;
|
|
using System.Diagnostics.CodeAnalysis;
|
|
using AcDream.Content;
|
|
using AcDream.Content.Vfx;
|
|
using AcDream.Core.Vfx;
|
|
using DatReaderWriter;
|
|
using DatReaderWriter.DBObjs;
|
|
using DatReaderWriter.Lib.IO;
|
|
using DatReaderWriter.Options;
|
|
using DatReaderWriter.Types;
|
|
using DatAnimation = DatReaderWriter.DBObjs.Animation;
|
|
using DatPhysicsScript = DatReaderWriter.DBObjs.PhysicsScript;
|
|
|
|
namespace AcDream.Content.Tests.Vfx;
|
|
|
|
public sealed class RetailDatLoaderTests
|
|
{
|
|
/// <summary>
|
|
/// Runs both callers on dedicated threads rather than thread-pool work
|
|
/// items.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// These tests measure whether the LOADER serialises reads, and the
|
|
/// interesting ones assert on <c>MaxConcurrentReads</c>. A pair of
|
|
/// <see cref="Task.Run(Func{int})"/> work items does not guarantee two
|
|
/// workers are ever in flight at once: on a saturated or low-core pool -
|
|
/// four cores on a hosted CI runner - the second item simply queues behind
|
|
/// the first, both 40 ms reads run back to back, and the assertion fails
|
|
/// for a reason that has nothing to do with the loader. Observed exactly
|
|
/// that way in CI on both operating systems, and reproduced locally by
|
|
/// pinning the suite to two CPUs (5 failures in 6).
|
|
/// <para><see cref="TaskCreationOptions.LongRunning"/> on the default
|
|
/// scheduler asks for a thread per callback, which makes the concurrency
|
|
/// the assertions measure actually available. The assertions themselves are
|
|
/// unchanged and still fail if the loader serialises - and the coalescing
|
|
/// cases get stronger, because two callers genuinely in flight is the
|
|
/// situation coalescing exists for, where a sequential pair only ever
|
|
/// exercised a cache hit.</para>
|
|
/// </remarks>
|
|
private static Task<T[]> InParallel<T>(Func<T> first, Func<T> second) =>
|
|
Task.WhenAll(
|
|
Task.Factory.StartNew(
|
|
first,
|
|
CancellationToken.None,
|
|
TaskCreationOptions.LongRunning,
|
|
TaskScheduler.Default),
|
|
Task.Factory.StartNew(
|
|
second,
|
|
CancellationToken.None,
|
|
TaskCreationOptions.LongRunning,
|
|
TaskScheduler.Default));
|
|
|
|
private sealed class RawDatabase : IDatDatabase
|
|
{
|
|
private readonly Dictionary<uint, byte[]> _entries = new();
|
|
private int _activeReads;
|
|
private int _maxConcurrentReads;
|
|
private int _totalReads;
|
|
|
|
public DatDatabase Db => null!;
|
|
public int Iteration => 0;
|
|
public int ReadDelayMilliseconds { get; set; }
|
|
public int MaxConcurrentReads => Volatile.Read(ref _maxConcurrentReads);
|
|
public int TotalReads => Volatile.Read(ref _totalReads);
|
|
public void Add(uint id, byte[] bytes) => _entries[id] = bytes;
|
|
public IEnumerable<uint> GetAllIdsOfType<T>() where T : IDBObj => _entries.Keys;
|
|
public bool TryGet<T>(uint fileId, [MaybeNullWhen(false)] out T value) where T : IDBObj
|
|
{
|
|
value = default;
|
|
return false;
|
|
}
|
|
public bool TryGetFileBytes(uint fileId, [MaybeNullWhen(false)] out byte[] value)
|
|
{
|
|
Interlocked.Increment(ref _totalReads);
|
|
int active = Interlocked.Increment(ref _activeReads);
|
|
int observed;
|
|
do
|
|
{
|
|
observed = Volatile.Read(ref _maxConcurrentReads);
|
|
}
|
|
while (active > observed
|
|
&& Interlocked.CompareExchange(ref _maxConcurrentReads, active, observed) != observed);
|
|
try
|
|
{
|
|
if (ReadDelayMilliseconds > 0)
|
|
Thread.Sleep(ReadDelayMilliseconds);
|
|
return _entries.TryGetValue(fileId, out value);
|
|
}
|
|
finally
|
|
{
|
|
Interlocked.Decrement(ref _activeReads);
|
|
}
|
|
}
|
|
public bool TryGetFileBytes(uint fileId, ref byte[] bytes, out int bytesRead)
|
|
{
|
|
if (!_entries.TryGetValue(fileId, out byte[]? value))
|
|
{
|
|
bytesRead = 0;
|
|
return false;
|
|
}
|
|
if (bytes.Length < value.Length)
|
|
bytes = new byte[value.Length];
|
|
value.CopyTo(bytes, 0);
|
|
bytesRead = value.Length;
|
|
return true;
|
|
}
|
|
public bool TrySave<T>(T obj, int iteration = 0) where T : IDBObj => false;
|
|
public void Dispose() { }
|
|
}
|
|
|
|
[Fact]
|
|
public void PhysicsScript_BlockingHookRetainsPayloadAndFollowingCursor()
|
|
{
|
|
DatPhysicsScript script = RetailPhysicsScriptLoader.Parse(
|
|
ProjectileVfxDatFixtures.PhysicsScriptCreateBlockingThenAnimationDone);
|
|
|
|
Assert.Equal(0x3300F001u, script.Id);
|
|
Assert.Equal(2, script.ScriptData.Count);
|
|
Assert.Equal(0.0, script.ScriptData[0].StartTime);
|
|
var blocking = Assert.IsType<RetailCreateBlockingParticleHook>(
|
|
script.ScriptData[0].Hook);
|
|
Assert.Equal(0x3200F001u, blocking.EmitterInfoId.DataId);
|
|
Assert.Equal(0xFFFFFFFFu, blocking.PartIndex);
|
|
Assert.Equal(new Vector3(1f, 2f, 3f), blocking.Offset.Origin);
|
|
Assert.Equal(new Quaternion(0.5f, -0.5f, -0.5f, 0.5f),
|
|
blocking.Offset.Orientation);
|
|
Assert.Equal(7u, blocking.EmitterId);
|
|
Assert.Equal(1.25, script.ScriptData[1].StartTime);
|
|
Assert.IsType<AnimationDoneHook>(script.ScriptData[1].Hook);
|
|
}
|
|
|
|
[Fact]
|
|
public void Animation_BlockingHookRetainsPayloadAndFollowingCursor()
|
|
{
|
|
DatAnimation animation = RetailAnimationLoader.Parse(
|
|
ProjectileVfxDatFixtures.AnimationCreateBlockingThenAnimationDone);
|
|
|
|
Assert.Equal(0x0300F001u, animation.Id);
|
|
AnimationFrame frame = Assert.Single(animation.PartFrames);
|
|
Assert.Equal(2, frame.Hooks.Count);
|
|
var blocking = Assert.IsType<RetailCreateBlockingParticleHook>(frame.Hooks[0]);
|
|
Assert.Equal(0x3200F002u, blocking.EmitterInfoId.DataId);
|
|
Assert.Equal(0u, blocking.PartIndex);
|
|
Assert.Equal(new Vector3(-1f, 4f, 0.25f), blocking.Offset.Origin);
|
|
Assert.Equal(Quaternion.Identity, blocking.Offset.Orientation);
|
|
Assert.Equal(9u, blocking.EmitterId);
|
|
Assert.IsType<AnimationDoneHook>(frame.Hooks[1]);
|
|
}
|
|
|
|
[Fact]
|
|
public void OrdinaryPhysicsScript_MatchesPackageDecoder()
|
|
{
|
|
byte[] bytes = ProjectileVfxDatFixtures.OrdinaryPhysicsScript;
|
|
DatPhysicsScript retail = RetailPhysicsScriptLoader.Parse(bytes);
|
|
var package = new DatPhysicsScript();
|
|
var reader = new DatBinReader(bytes);
|
|
Assert.True(package.Unpack(reader));
|
|
|
|
Assert.Equal(reader.Length, reader.Offset);
|
|
Assert.Equal(package.Id, retail.Id);
|
|
Assert.Equal(package.ScriptData.Count, retail.ScriptData.Count);
|
|
Assert.Equal(package.ScriptData[0].StartTime, retail.ScriptData[0].StartTime);
|
|
Assert.Equal(package.ScriptData[0].Hook.HookType, retail.ScriptData[0].Hook.HookType);
|
|
Assert.Equal(package.ScriptData[0].Hook.Direction, retail.ScriptData[0].Hook.Direction);
|
|
}
|
|
|
|
[Fact]
|
|
public void OrdinaryAnimation_MatchesPackageDecoder()
|
|
{
|
|
byte[] bytes = ProjectileVfxDatFixtures.OrdinaryAnimation;
|
|
DatAnimation retail = RetailAnimationLoader.Parse(bytes);
|
|
var package = new DatAnimation();
|
|
var reader = new DatBinReader(bytes);
|
|
Assert.True(package.Unpack(reader));
|
|
|
|
Assert.Equal(reader.Length, reader.Offset);
|
|
Assert.Equal(package.Id, retail.Id);
|
|
Assert.Equal(package.Flags, retail.Flags);
|
|
Assert.Equal(package.NumParts, retail.NumParts);
|
|
Assert.Equal(package.PartFrames.Count, retail.PartFrames.Count);
|
|
Assert.Equal(package.PartFrames[0].Hooks[0].HookType,
|
|
retail.PartFrames[0].Hooks[0].HookType);
|
|
Assert.Equal(package.PartFrames[0].Hooks[0].Direction,
|
|
retail.PartFrames[0].Hooks[0].Direction);
|
|
}
|
|
|
|
[Fact]
|
|
public void PhysicsScript_SortsHooksByRetailStartTimeAfterDecode()
|
|
{
|
|
byte[] bytes = ProjectileVfxDatFixtures.PhysicsScriptCreateBlockingThenAnimationDone.ToArray();
|
|
// The fixture stores starts 0.0 then 1.25. Make the first later than
|
|
// the second without changing either hook payload or cursor.
|
|
System.Buffers.Binary.BinaryPrimitives.WriteDoubleLittleEndian(
|
|
bytes.AsSpan(8), 2.0);
|
|
|
|
DatPhysicsScript script = RetailPhysicsScriptLoader.Parse(bytes);
|
|
|
|
Assert.Equal(1.25, script.ScriptData[0].StartTime);
|
|
Assert.IsType<AnimationDoneHook>(script.ScriptData[0].Hook);
|
|
Assert.Equal(2.0, script.ScriptData[1].StartTime);
|
|
Assert.IsType<RetailCreateBlockingParticleHook>(script.ScriptData[1].Hook);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnimationDidValidation_UsesRetailHighByte()
|
|
{
|
|
Assert.True(RetailAnimationLoader.IsAnimationDid(0x03010000u));
|
|
Assert.True(RetailAnimationLoader.IsAnimationDid(0x03FFFFFFu));
|
|
Assert.False(RetailAnimationLoader.IsAnimationDid(0x04010000u));
|
|
}
|
|
|
|
[Fact]
|
|
public void Loaders_AcceptHighIndexesCacheIdentityAndRejectEmbeddedIdMismatch()
|
|
{
|
|
const uint scriptDid = 0x33010000u;
|
|
const uint animationDid = 0x03010000u;
|
|
var portal = new RawDatabase();
|
|
byte[] scriptBytes = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
|
|
byte[] animationBytes = ProjectileVfxDatFixtures.OrdinaryAnimation.ToArray();
|
|
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(scriptBytes, scriptDid);
|
|
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(animationBytes, animationDid);
|
|
portal.Add(scriptDid, scriptBytes);
|
|
portal.Add(animationDid, animationBytes);
|
|
|
|
var scripts = new RetailPhysicsScriptLoader(portal);
|
|
var animations = new RetailAnimationLoader(portal);
|
|
DatPhysicsScript script = Assert.IsType<DatPhysicsScript>(scripts.LoadPhysicsScript(scriptDid));
|
|
DatAnimation animation = Assert.IsType<DatAnimation>(animations.LoadAnimation(animationDid));
|
|
Assert.Same(script, scripts.LoadPhysicsScript(scriptDid));
|
|
Assert.Same(animation, animations.LoadAnimation(animationDid));
|
|
|
|
const uint wrongScriptKey = 0x33010001u;
|
|
const uint wrongAnimationKey = 0x03010001u;
|
|
portal.Add(wrongScriptKey, scriptBytes);
|
|
portal.Add(wrongAnimationKey, animationBytes);
|
|
Assert.Throws<InvalidDataException>(() => scripts.LoadPhysicsScript(wrongScriptKey));
|
|
Assert.Throws<InvalidDataException>(() => animations.LoadAnimation(wrongAnimationKey));
|
|
}
|
|
|
|
[Fact]
|
|
public void AnimationCache_CountPressureEvictsLruWithoutInvalidatingLiveReferences()
|
|
{
|
|
const uint firstDid = 0x03010020u;
|
|
const uint secondDid = 0x03010021u;
|
|
const uint thirdDid = 0x03010022u;
|
|
var portal = new RawDatabase();
|
|
portal.Add(firstDid, AnimationBytes(firstDid));
|
|
portal.Add(secondDid, AnimationBytes(secondDid));
|
|
portal.Add(thirdDid, AnimationBytes(thirdDid));
|
|
var loader = new RetailAnimationLoader(
|
|
portal,
|
|
maximumEstimatedBytes: long.MaxValue,
|
|
maximumEntries: 2);
|
|
|
|
DatAnimation first = Assert.IsType<DatAnimation>(
|
|
loader.LoadAnimation(firstDid));
|
|
_ = Assert.IsType<DatAnimation>(loader.LoadAnimation(secondDid));
|
|
_ = Assert.IsType<DatAnimation>(loader.LoadAnimation(thirdDid));
|
|
|
|
AnimationCacheDiagnostics pressured = loader.Diagnostics;
|
|
Assert.Equal(2, pressured.Count);
|
|
Assert.Equal(1, pressured.Stats.Evictions);
|
|
Assert.Equal(firstDid, first.Id);
|
|
|
|
DatAnimation reloaded = Assert.IsType<DatAnimation>(
|
|
loader.LoadAnimation(firstDid));
|
|
Assert.NotSame(first, reloaded);
|
|
Assert.Equal(4, portal.TotalReads);
|
|
Assert.Equal(2, loader.Diagnostics.Stats.Evictions);
|
|
}
|
|
|
|
[Fact]
|
|
public void AnimationCache_OversizeEntryIsServedButNeverRetained()
|
|
{
|
|
const uint animationDid = 0x03010023u;
|
|
var portal = new RawDatabase();
|
|
portal.Add(animationDid, AnimationBytes(animationDid));
|
|
var loader = new RetailAnimationLoader(
|
|
portal,
|
|
maximumEstimatedBytes: 1,
|
|
maximumEntries: 2);
|
|
|
|
Assert.NotNull(loader.LoadAnimation(animationDid));
|
|
Assert.NotNull(loader.LoadAnimation(animationDid));
|
|
|
|
Assert.Equal(0, loader.Diagnostics.Count);
|
|
Assert.Equal(0, loader.Diagnostics.EstimatedBytes);
|
|
Assert.Equal(2, portal.TotalReads);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AnimationCache_CoalescesSameDidAndAllowsUnrelatedReadsInParallel()
|
|
{
|
|
const uint firstDid = 0x03010024u;
|
|
const uint secondDid = 0x03010025u;
|
|
var portal = new RawDatabase { ReadDelayMilliseconds = 40 };
|
|
portal.Add(firstDid, AnimationBytes(firstDid));
|
|
portal.Add(secondDid, AnimationBytes(secondDid));
|
|
var loader = new RetailAnimationLoader(portal);
|
|
|
|
DatAnimation?[] firstPair = await InParallel(
|
|
() => loader.LoadAnimation(firstDid),
|
|
() => loader.LoadAnimation(firstDid));
|
|
Assert.Same(firstPair[0], firstPair[1]);
|
|
Assert.Equal(1, portal.TotalReads);
|
|
|
|
await InParallel(
|
|
() => loader.LoadAnimation(secondDid),
|
|
() => loader.LoadAnimation(0x03010026u));
|
|
|
|
Assert.True(portal.MaxConcurrentReads >= 2);
|
|
Assert.Equal(3, portal.TotalReads);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PhysicsScriptLoader_AllowsConcurrentFirstReads()
|
|
{
|
|
const uint firstDid = 0x33010010u;
|
|
const uint secondDid = 0x33010011u;
|
|
var portal = new RawDatabase { ReadDelayMilliseconds = 40 };
|
|
byte[] first = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
|
|
byte[] second = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
|
|
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(first, firstDid);
|
|
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(second, secondDid);
|
|
portal.Add(firstDid, first);
|
|
portal.Add(secondDid, second);
|
|
var loader = new RetailPhysicsScriptLoader(portal);
|
|
|
|
await InParallel(
|
|
() => loader.LoadPhysicsScript(firstDid),
|
|
() => loader.LoadPhysicsScript(secondDid));
|
|
|
|
Assert.Equal(2, portal.MaxConcurrentReads);
|
|
Assert.Equal(2, portal.TotalReads);
|
|
}
|
|
|
|
private static byte[] AnimationBytes(uint id)
|
|
{
|
|
byte[] bytes =
|
|
ProjectileVfxDatFixtures.OrdinaryAnimation.ToArray();
|
|
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(
|
|
bytes,
|
|
id);
|
|
return bytes;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PhysicsScriptLoader_CoalescesConcurrentReadsForTheSameDid()
|
|
{
|
|
const uint scriptDid = 0x33010012u;
|
|
var portal = new RawDatabase { ReadDelayMilliseconds = 40 };
|
|
byte[] bytes = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
|
|
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(bytes, scriptDid);
|
|
portal.Add(scriptDid, bytes);
|
|
var loader = new RetailPhysicsScriptLoader(portal);
|
|
|
|
DatPhysicsScript?[] loaded = await InParallel(
|
|
() => loader.LoadPhysicsScript(scriptDid),
|
|
() => loader.LoadPhysicsScript(scriptDid));
|
|
|
|
Assert.All(loaded, Assert.NotNull);
|
|
Assert.Same(loaded[0], loaded[1]);
|
|
Assert.Equal(1, portal.TotalReads);
|
|
}
|
|
|
|
[Fact]
|
|
public void Parsers_RejectTrailingOrTruncatedEntries()
|
|
{
|
|
byte[] script = ProjectileVfxDatFixtures.PhysicsScriptCreateBlockingThenAnimationDone;
|
|
byte[] animation = ProjectileVfxDatFixtures.AnimationCreateBlockingThenAnimationDone;
|
|
|
|
Assert.ThrowsAny<Exception>(() =>
|
|
RetailPhysicsScriptLoader.Parse(script.AsMemory(0, script.Length - 1)));
|
|
Assert.ThrowsAny<Exception>(() =>
|
|
RetailAnimationLoader.Parse(animation.AsMemory(0, animation.Length - 1)));
|
|
Assert.Throws<InvalidDataException>(() =>
|
|
RetailPhysicsScriptLoader.Parse(script.Concat(new byte[] { 0 }).ToArray()));
|
|
Assert.Throws<InvalidDataException>(() =>
|
|
RetailAnimationLoader.Parse(animation.Concat(new byte[] { 0 }).ToArray()));
|
|
}
|
|
|
|
[Fact]
|
|
public void PhysicsScriptParserRejectsNonFiniteHookTime()
|
|
{
|
|
byte[] script = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
|
|
System.Buffers.Binary.BinaryPrimitives.WriteInt64LittleEndian(
|
|
script.AsSpan(8, 8),
|
|
BitConverter.DoubleToInt64Bits(double.NaN));
|
|
|
|
Assert.Throws<InvalidDataException>(() => RetailPhysicsScriptLoader.Parse(script));
|
|
}
|
|
|
|
[Fact]
|
|
public void Parsers_RejectUnsupportedHookTypesWithoutReturningPartialObjects()
|
|
{
|
|
byte[] script = ProjectileVfxDatFixtures.OrdinaryPhysicsScript.ToArray();
|
|
byte[] animation = ProjectileVfxDatFixtures.OrdinaryAnimation.ToArray();
|
|
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(
|
|
script.AsSpan(16), 0xDEADBEEFu);
|
|
System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(
|
|
animation.AsSpan(20), 0xDEADBEEFu);
|
|
|
|
Assert.ThrowsAny<Exception>(() => RetailPhysicsScriptLoader.Parse(script));
|
|
Assert.ThrowsAny<Exception>(() => RetailAnimationLoader.Parse(animation));
|
|
}
|
|
|
|
[Fact]
|
|
public void InstalledBlockingScripts_ParseThroughRetailShape_WhenDatsAvailable()
|
|
{
|
|
string? datDir = ResolveDatDirectory();
|
|
if (datDir is null)
|
|
return;
|
|
|
|
using var dats = new DatCollection(datDir, DatAccessType.Read);
|
|
using var boundedDats = new DatCollectionAdapter(dats);
|
|
var loader = new RetailPhysicsScriptLoader(boundedDats);
|
|
|
|
foreach (uint id in new[] { 0x33000AEAu, 0x33000AF8u })
|
|
{
|
|
DatPhysicsScript script = Assert.IsType<DatPhysicsScript>(
|
|
loader.LoadPhysicsScript(id));
|
|
Assert.Contains(script.ScriptData,
|
|
item => item.Hook is RetailCreateBlockingParticleHook);
|
|
}
|
|
}
|
|
|
|
private static string? ResolveDatDirectory()
|
|
{
|
|
string? configured = System.Environment.GetEnvironmentVariable("ACDREAM_DAT_DIR");
|
|
if (!string.IsNullOrWhiteSpace(configured)
|
|
&& File.Exists(Path.Combine(configured, "client_portal.dat")))
|
|
{
|
|
return configured;
|
|
}
|
|
|
|
const string installed = @"C:\Turbine\Asheron's Call";
|
|
return File.Exists(Path.Combine(installed, "client_portal.dat"))
|
|
? installed
|
|
: null;
|
|
}
|
|
}
|