test(headless): enforce 30-root K4 endurance

tests/AcDream.Headless.Tests/HeadlessProcessContentOwnerTests.cs

@@ -82,10 +82,14 @@ public sealed class HeadlessProcessContentOwnerTests
         Assert.Equal(1, factory.Dats.DisposeSuccessCount);
     }
 
-    [Fact]
+    [Theory]
-    public void FiveSessionsShareContentButNotMutableRuntimeOwners()
+    [InlineData(1)]
+    [InlineData(5)]
+    [InlineData(10)]
+    [InlineData(30)]
+    public void SessionsShareContentButNotMutableRuntimeOwners(
+        int sessionCount)
     {
-        const int sessionCount = 5;
         var factory = new FixtureContentFactory();
         var configuration = new HeadlessConfiguration
         {
@@ -103,11 +107,12 @@ public sealed class HeadlessProcessContentOwnerTests
             Enumerable.Repeat(
                 "fixture-password" + Environment.NewLine,
                 sessionCount)));
+        using var diagnostics = new StringWriter();
         using var host = new HeadlessProcessHost(
             configuration,
             HeadlessPathSet.Resolve(new HeadlessPathOverrides()),
             input,
-            TextWriter.Null,
+            diagnostics,
             contentFactory: factory);
 
         Assert.Equal(1, factory.OpenCount);
@@ -179,6 +184,23 @@ public sealed class HeadlessProcessContentOwnerTests
                     session.Runtime.CaptureOwnership().IsConverged));
         Assert.Equal(1, factory.Prepared.DisposeSuccessCount);
         Assert.Equal(1, factory.Dats.DisposeSuccessCount);
+        string output = diagnostics.ToString();
+        Assert.Contains(
+            "\"state\":\"disposed\"",
+            output,
+            StringComparison.Ordinal);
+        Assert.Contains(
+            $"\"convergedRuntimeCount\":{sessionCount}",
+            output,
+            StringComparison.Ordinal);
+        Assert.Contains(
+            "\"isConverged\":true",
+            output,
+            StringComparison.Ordinal);
+        Assert.DoesNotContain(
+            "fixture-password",
+            output,
+            StringComparison.Ordinal);
     }
 
     private static HeadlessContentDescriptor ContentDescriptor() => new()

tests/AcDream.Headless.Tests/HeadlessSessionIsolationTests.cs

@@ -9,16 +9,27 @@ using AcDream.Headless.Diagnostics;
 using AcDream.Headless.Hosting;
 using AcDream.Runtime;
 using AcDream.Runtime.Entities;
+using AcDream.Runtime.Gameplay;
 using AcDream.Runtime.Session;
 
 namespace AcDream.Headless.Tests;
 
+[CollectionDefinition(
+    HeadlessEnduranceCollection.Name,
+    DisableParallelization = true)]
+public sealed class HeadlessEnduranceCollection
+{
+    public const string Name = "Headless endurance";
+}
+
+[Collection(HeadlessEnduranceCollection.Name)]
 public sealed class HeadlessSessionIsolationTests
 {
     [Theory]
     [InlineData(1)]
     [InlineData(5)]
     [InlineData(10)]
+    [InlineData(30)]
     public void SameServerIdentityStaysIsolatedAcrossActivityPortalAndReconnect(
         int sessionCount)
     {
@@ -123,9 +134,211 @@ public sealed class HeadlessSessionIsolationTests
                 Assert.True(host.Runtime.CaptureOwnership().IsConverged));
     }
 
+    [Fact]
+    public void ThirtySessionMixedWorkloadMaintainsIsolationAndConverges()
+    {
+        const int sessionCount = 30;
+        const int turnPeriodMilliseconds = 15;
+        const int simulatedSeconds = 2 * 60 * 60;
+        const int frameCount =
+            simulatedSeconds * 1000 / turnPeriodMilliseconds;
+        const long maximumProcessAllocationBytesPerSecond =
+            384L * 1024L;
+        const uint sharedPlayerGuid = 0x50000001u;
+        var time = new ManualTimeProvider();
+        var operations = new FixtureOperations(sharedPlayerGuid);
+        HeadlessSessionHost[] hosts = Enumerable.Range(0, sessionCount)
+            .Select(index => CreateHost(index, operations, time))
+            .ToArray();
+        var reconnectCounts = new int[sessionCount];
+        var random = new Random(0xAC2013);
+        try
+        {
+            foreach (HeadlessSessionHost host in hosts)
+            {
+                Assert.Equal(
+                    RuntimeSessionStartStatus.Connected,
+                    host.Start().Status);
+                WorldSession session =
+                    operations.ActiveSession(host.SessionId);
+                session.GameActionCapture = _ => { };
+                SpawnInto(session, sharedPlayerGuid, 1f);
+                Assert.True(
+                    host.Commands.Selection.SelectObject(
+                        host.Runtime.Generation,
+                        sharedPlayerGuid).Accepted);
+                Assert.True(
+                    host.Commands.Movement.SetIntent(
+                        host.Runtime.Generation,
+                        new MovementInput(
+                            Forward: true,
+                            Run: true)).Accepted);
+            }
+
+            var scheduler = new HeadlessProcessScheduler(
+                hosts,
+                time);
+            scheduler.RebaseDeadlinesAfterSessionStart();
+            long retainedBefore = GC.GetTotalMemory(
+                forceFullCollection: true);
+            long allocatedBefore =
+                GC.GetAllocatedBytesForCurrentThread();
+            for (int frame = 1; frame <= frameCount; frame++)
+            {
+                if (frame % 211 == 0)
+                {
+                    int index = random.Next(sessionCount);
+                    Portal(
+                        operations.ActiveSession(
+                            hosts[index].SessionId),
+                        sharedPlayerGuid,
+                        index + frame / 100f,
+                        checked((ushort)(
+                            2 + reconnectCounts[index]
+                            + (frame / 211))));
+                }
+                if (frame % 353 == 0)
+                {
+                    int index = random.Next(sessionCount);
+                    HeadlessSessionHost host = hosts[index];
+                    Assert.True(
+                        host.Commands.Chat.Execute(
+                            host.Runtime.Generation,
+                            new RuntimeChatCommand(
+                                RuntimeChatChannel.Say,
+                                $"stress-{frame}-{index}")).Accepted);
+                    _ = host.Commands.Selection.Execute(
+                        host.Runtime.Generation,
+                        RuntimeSelectionCommand.UseSelected);
+                    _ = host.Commands.Combat.Execute(
+                        host.Runtime.Generation,
+                        RuntimeCombatCommand.ToggleMode);
+                    _ = host.Commands.Magic.Execute(
+                        host.Runtime.Generation,
+                        new RuntimeMagicCommand(1u));
+                }
+                if (frame % 997 == 0)
+                {
+                    int index = random.Next(sessionCount);
+                    HeadlessSessionHost host = hosts[index];
+                    ulong previousGeneration =
+                        host.Runtime.Generation.Value;
+                    Assert.Equal(
+                        RuntimeSessionStartStatus.Connected,
+                        host.Reconnect().Status);
+                    reconnectCounts[index]++;
+                    Assert.True(
+                        host.Runtime.Generation.Value
+                        > previousGeneration);
+                    WorldSession replacement =
+                        operations.ActiveSession(host.SessionId);
+                    replacement.GameActionCapture = _ => { };
+                    SpawnInto(
+                        replacement,
+                        sharedPlayerGuid,
+                        index + frame);
+                    Assert.True(
+                        host.Commands.Selection.SelectObject(
+                            host.Runtime.Generation,
+                            sharedPlayerGuid).Accepted);
+                    Assert.True(
+                        host.Commands.Movement.SetIntent(
+                            host.Runtime.Generation,
+                            new MovementInput(
+                                Forward: (frame & 1) == 0,
+                                TurnRight: (frame & 1) != 0,
+                                Run: true)).Accepted);
+                }
+
+                time.Advance(
+                    TimeSpan.FromMilliseconds(
+                        turnPeriodMilliseconds));
+                if (!scheduler.DispatchDue(time.GetTimestamp()))
+                {
+                    throw new InvalidOperationException(
+                        $"No session dispatched at stress frame {frame}.");
+                }
+            }
+            long allocatedBytes =
+                GC.GetAllocatedBytesForCurrentThread()
+                - allocatedBefore;
+            long retainedAfter = GC.GetTotalMemory(
+                forceFullCollection: true);
+            long retainedGrowth = Math.Max(
+                0L,
+                retainedAfter - retainedBefore);
+
+            HeadlessSchedulerSnapshot snapshot =
+                scheduler.CaptureSnapshot();
+            Assert.Equal(
+                (long)sessionCount * frameCount,
+                snapshot.TurnCount);
+            Assert.Equal(0L, snapshot.CatchUpCollapseCount);
+            Assert.Equal(0L, snapshot.LateDeadlineCount);
+            Assert.Equal(0, snapshot.FaultedSessionCount);
+            Assert.Equal(sessionCount, snapshot.ActiveSessionCount);
+            Assert.InRange(
+                allocatedBytes,
+                0L,
+                maximumProcessAllocationBytesPerSecond
+                * simulatedSeconds);
+            Assert.InRange(
+                retainedGrowth,
+                0L,
+                16L * 1024L * 1024L);
+            Assert.All(
+                reconnectCounts,
+                static count => Assert.True(count > 0));
+            var finalRecords = new List<RuntimeEntityRecord>(
+                sessionCount);
+            Assert.All(
+                hosts,
+                host =>
+                {
+                    Assert.Equal(
+                        (ulong)frameCount,
+                        host.Runtime.Clock.FrameNumber);
+                    Assert.True(
+                        host.Runtime.EntityObjects.Entities.TryGetActive(
+                            sharedPlayerGuid,
+                            out RuntimeEntityRecord finalRecord));
+                    finalRecords.Add(finalRecord);
+                    Assert.True(
+                        host.Runtime.TransitOwner
+                            .CaptureOwnership()
+                            .IsSessionIdle);
+                });
+            Assert.Equal(
+                sessionCount,
+                finalRecords
+                    .Distinct(
+                        ReferenceEqualityComparer.Instance)
+                    .Count());
+        }
+        finally
+        {
+            for (int index = hosts.Length - 1; index >= 0; index--)
+                hosts[index].Dispose();
+        }
+
+        int reconnectCount = reconnectCounts.Sum();
+        Assert.Equal(
+            sessionCount + reconnectCount,
+            operations.CreatedSessionCount);
+        Assert.Equal(
+            sessionCount + reconnectCount,
+            operations.DisposedSessionCount);
+        Assert.All(
+            hosts,
+            static host =>
+                Assert.True(
+                    host.Runtime.CaptureOwnership().IsConverged));
+    }
+
     private static HeadlessSessionHost CreateHost(
         int index,
-        FixtureOperations operations)
+        FixtureOperations operations,
+        TimeProvider? timeProvider = null)
     {
         string id = $"bot-{index}";
         var credential = new HeadlessCredentialSecret(
@@ -160,7 +373,8 @@ public sealed class HeadlessSessionIsolationTests
                 },
                 credential,
                 new HeadlessDiagnosticWriter(TextWriter.Null),
-                operations);
+                operations,
+                timeProvider);
         }
         catch
         {
@@ -181,11 +395,12 @@ public sealed class HeadlessSessionIsolationTests
     private static void Portal(
         WorldSession session,
         uint guid,
-        float positionX)
+        float positionX,
+        ushort teleportSequence = 1)
     {
         EventDelegate<Action<uint>>(
             session,
-            nameof(session.TeleportStarted))(1u);
+            nameof(session.TeleportStarted))(teleportSequence);
         EventDelegate<Action<WorldSession.EntityPositionUpdate>>(
             session,
             nameof(session.PositionUpdated))(
@@ -199,8 +414,9 @@ public sealed class HeadlessSessionIsolationTests
                     PlacementId: null,
                     IsGrounded: true,
                     InstanceSequence: 1,
-                    PositionSequence: 2,
+                    PositionSequence: checked(
-                    TeleportSequence: 1,
+                        (ushort)(teleportSequence + 1)),
+                    TeleportSequence: teleportSequence,
                     ForcePositionSequence: 0));
     }
 
@@ -342,6 +558,19 @@ public sealed class HeadlessSessionIsolationTests
         }
     }
 
+    private sealed class ManualTimeProvider : TimeProvider
+    {
+        private long _timestamp;
+
+        public override long TimestampFrequency =>
+            TimeSpan.TicksPerSecond;
+
+        public override long GetTimestamp() => _timestamp;
+
+        internal void Advance(TimeSpan duration) =>
+            _timestamp = checked(_timestamp + duration.Ticks);
+    }
+
     private sealed class FixtureTransport : IWorldSessionTransport
     {
         public void Send(ReadOnlySpan<byte> datagram)