fix(physics): restore nested per-cell collision retries

src/AcDream.Core/Physics/PhysicsEngine.cs

@@ -63,6 +63,18 @@ public sealed class PhysicsEngine
     /// </summary>
     public Action<string>? DiagnosticLog { get; set; }
 
+    /// <summary>
+    /// Deterministic test seam for the retail per-cell dispatcher. Production
+    /// leaves this null. Tests may observe a completed phase and substitute its
+    /// returned state to prove retry/order semantics without geometry-specific
+    /// response coupling.
+    /// </summary>
+    internal Func<
+        TransitionCellCollisionPhase,
+        uint,
+        TransitionState,
+        TransitionState>? TransitionCellCollisionTestHook { get; set; }
+
     /// <summary>
     /// True once the landblock covering <paramref name="cellOrLandblockId"/> has had its
     /// terrain + cells registered via <see cref="AddLandblock"/>. Accepts a canonical

src/AcDream.Core/Physics/TransitionTypes.cs

@@ -15,6 +15,18 @@ public enum TransitionState
     Slid = 4,
 }
 
+/// <summary>
+/// Test-observation boundary for retail's atomic per-cell collision pass.
+/// Production never installs the corresponding hook on
+/// <see cref="PhysicsEngine"/>.
+/// </summary>
+internal enum TransitionCellCollisionPhase
+{
+    Environment,
+    Building,
+    Objects,
+}
+
 public enum InsertType
 {
     Transition = 0,
@@ -1685,14 +1697,18 @@ public sealed class Transition
     // -----------------------------------------------------------------------
 
     /// <summary>
-    /// ACE Transition.TransitionalInsert — retry loop for collision resolution.
+    /// Retail <c>CTransition::transitional_insert</c> (0x0050B6F0) — the
+    /// outer transition retry loop. Each outer attempt delegates the complete
+    /// primary-cell transaction to <see cref="InsertIntoCell"/>, which owns a
+    /// second retry budget matching <c>CTransition::insert_into_cell</c>
+    /// (0x00509E70).
     ///
     /// <para>
-    /// Per ACE: iterate up to numAttempts times. Each iteration runs the full
+    /// Each outer attempt asks <see cref="InsertIntoCell"/> to retry the atomic
-    /// collision pipeline (env + objects) at the current CheckPos. The pipeline
+    /// environment → building → objects composition up to
-    /// can MUTATE CheckPos (push-out, slide). On Slid/Adjusted, clear state and
+    /// <paramref name="numAttempts"/> times. An Adjusted/Slid result that
-    /// retry — the next iteration tests the NEW CheckPos against all nearby
+    /// exhausts that inner budget is then eligible for another outer attempt.
-    /// objects again, which catches "slide into a second wall" corner cases.
+    /// Retail therefore permits up to N×N primary-cell passes, not N total.
     /// </para>
     ///
     /// <para>
@@ -1730,18 +1746,21 @@ public sealed class Transition
 
         for (int attempt = 0; attempt < numAttempts; attempt++)
         {
-            // ── Phase 1: environment collision (terrain + indoor BSP) ───
+            transitState = InsertIntoCell(
-            // Primary cell only — retail CEnvCell/CLandCell::find_collisions
+                engine,
-            // step 1 (find_env_collisions). Other cells run in Phase 2.5.
+                sp.CheckCellId,
-            transitState = FindEnvCollisions(engine);
+                numAttempts);
 
             if (transitState == TransitionState.Collided)
+            {
+                sp.NegPolyHit = false;
                 return TransitionState.Collided;
+            }
 
             if (transitState == TransitionState.Slid)
             {
-                // Env collision slid the sphere. Clear state and retry at
+                // Retail transitional_insert repeats the Slid contact clear
-                // the new CheckPos to see if we hit anything else.
+                // at its outer boundary, then clears neg_poly_hit.
                 ci.ContactPlaneValid = false;
                 ci.ContactPlaneIsWater = false;
                 sp.NegPolyHit = false;
@@ -1750,70 +1769,15 @@ public sealed class Transition
 
             if (transitState == TransitionState.Adjusted)
             {
-                // Env modified CheckPos. Retry at new position.
+                // insert_into_cell exhausted its own retry budget. Preserve
+                // every sphere field except neg_poly_hit at this outer
+                // boundary and start the next outer attempt.
                 sp.NegPolyHit = false;
                 continue;
             }
 
-            // ── Phase 1b: the building channel (BR-7 / A6.P4) ───────────
+            if (transitState != TransitionState.OK)
-            // CLandCell::find_collisions (Ghidra 0x00532d60) interposes
-            // CSortCell::find_collisions (0x005340a0 — the per-LandCell
-            // building shell BSP) between env and objects. Indoor primary
-            // cells have no building leg (CEnvCell::find_collisions,
-            // 0x0052c100). No-op when the cell has no building.
-            var bldgState = FindBuildingCollisions(engine, sp.CheckCellId);
-
-            if (bldgState == TransitionState.Collided)
-                return TransitionState.Collided;
-
-            if (bldgState == TransitionState.Slid)
-            {
-                transitState = bldgState;
-                ci.ContactPlaneValid = false;
-                ci.ContactPlaneIsWater = false;
-                sp.NegPolyHit = false;
                 continue;
-            }
-
-            if (bldgState == TransitionState.Adjusted)
-            {
-                transitState = bldgState;
-                sp.NegPolyHit = false;
-                continue;
-            }
-
-            // ── Phase 2: object collision — PRIMARY cell's shadow list ──
-            // Retail CObjCell::find_obj_collisions(this) (0x0052b750), the
-            // tail of the primary cell's find_collisions. Other cells'
-            // lists run per cell in Phase 2.5 (check_other_cells).
-            var objState = FindObjCollisionsInCell(engine, sp.CheckCellId);
-            // L.4-diag: log Phase outcomes per attempt so we can see whether
-            // we're escaping to the step-down branch or churning in retries.
-            DumpPhase2(attempt, transitState, objState);
-
-            if (objState == TransitionState.Collided)
-                return TransitionState.Collided;
-
-            if (objState == TransitionState.Slid)
-            {
-                // Object collision applied a push-out and set sliding normal.
-                // Retry at the new CheckPos — we may have slid into another
-                // object, or need to re-verify env at the new position.
-                transitState = objState;
-                ci.ContactPlaneValid = false;
-                ci.ContactPlaneIsWater = false;
-                sp.NegPolyHit = false;
-                continue;
-            }
-
-            if (objState == TransitionState.Adjusted)
-            {
-                // Object modified CheckPos (e.g. PerfectClip adjust_to_plane).
-                // Retry at the new position.
-                transitState = objState;
-                sp.NegPolyHit = false;
-                continue;
-            }
 
             // ── Phase 2.5: other cells + carried-cell advance ────────────
             // Retail transitional_insert OK_TS case (0x0050b756): on a clean
@@ -2151,6 +2115,82 @@ public sealed class Transition
         return transitState;
     }
 
+    /// <summary>
+    /// Retail <c>CTransition::insert_into_cell</c> (0x00509E70). The cell's
+    /// virtual <c>find_collisions</c> call is one atomic environment →
+    /// building → objects transaction. Adjusted retries the entire
+    /// transaction. Slid additionally clears only contact-plane validity and
+    /// water state before retrying. OK and Collided return immediately.
+    /// </summary>
+    private TransitionState InsertIntoCell(
+        PhysicsEngine engine,
+        uint cellId,
+        int numAttempts)
+    {
+        if (cellId == 0)
+            return TransitionState.Collided;
+
+        TransitionState state = TransitionState.OK;
+        for (int attempt = 0; attempt < numAttempts; attempt++)
+        {
+            state = FindPrimaryCellCollisions(engine, cellId, attempt);
+            if (state is TransitionState.OK or TransitionState.Collided)
+                return state;
+
+            if (state == TransitionState.Slid)
+            {
+                CollisionInfo.ContactPlaneValid = false;
+                CollisionInfo.ContactPlaneIsWater = false;
+            }
+        }
+
+        return state;
+    }
+
+    /// <summary>
+    /// Primary-cell virtual <c>find_collisions</c> composition. A non-OK
+    /// response terminates this pass, so an inner retry always restarts from
+    /// environment before revisiting the building and object channels.
+    /// </summary>
+    private TransitionState FindPrimaryCellCollisions(
+        PhysicsEngine engine,
+        uint cellId,
+        int innerAttempt)
+    {
+        TransitionState environment = ObservePrimaryCellPhase(
+            engine,
+            TransitionCellCollisionPhase.Environment,
+            cellId,
+            FindEnvCollisions(engine));
+        if (environment != TransitionState.OK)
+            return environment;
+
+        TransitionState building = ObservePrimaryCellPhase(
+            engine,
+            TransitionCellCollisionPhase.Building,
+            cellId,
+            FindBuildingCollisions(engine, cellId));
+        if (building != TransitionState.OK)
+            return building;
+
+        TransitionState objects = ObservePrimaryCellPhase(
+            engine,
+            TransitionCellCollisionPhase.Objects,
+            cellId,
+            FindObjCollisionsInCell(engine, cellId));
+        DumpPhase2(innerAttempt, environment, objects);
+        return objects;
+    }
+
+    private static TransitionState ObservePrimaryCellPhase(
+        PhysicsEngine engine,
+        TransitionCellCollisionPhase phase,
+        uint cellId,
+        TransitionState actual)
+        => engine.TransitionCellCollisionTestHook is { } hook
+            ? hook(phase, cellId, actual)
+            : actual;
+
     private TransitionState EdgeSlideAfterStepDownFailed(
         PhysicsEngine engine,
         float stepDownHeight,

tests/AcDream.Core.Tests/Physics/TransitionInsertIntoCellRetryTests.cs

@@ -0,0 +1,169 @@
+using System;
+using System.Collections.Generic;
+using System.Numerics;
+using AcDream.Core.Physics;
+using DatReaderWriter.Types;
+using Xunit;
+
+namespace AcDream.Core.Tests.Physics;
+
+/// <summary>
+/// Pins retail's two-level collision retry structure:
+/// <c>transitional_insert(N)</c> wraps <c>insert_into_cell(N)</c>, and every
+/// inner retry restarts the complete cell transaction in
+/// environment → building → objects order.
+/// </summary>
+public sealed class TransitionInsertIntoCellRetryTests
+{
+    private const uint Landblock = 0xA9B40000u;
+    private const uint Cell = 0xA9B40001u;
+    private const uint ShellGfxObj = 0x0100F001u;
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public void NestedRetry_RecomputesAtomicCellPipeline_AndExceedsOuterBudget(
+        bool preparedFlat)
+    {
+        PhysicsEngine engine = BuildEngine(preparedFlat);
+        var phases = new List<TransitionCellCollisionPhase>();
+        int environmentCalls = 0;
+        int buildingCalls = 0;
+        int objectCalls = 0;
+
+        engine.TransitionCellCollisionTestHook = (phase, cellId, actual) =>
+        {
+            Assert.Equal(Cell, cellId);
+            Assert.Equal(TransitionState.OK, actual);
+            phases.Add(phase);
+
+            switch (phase)
+            {
+                case TransitionCellCollisionPhase.Environment:
+                    environmentCalls++;
+                    return environmentCalls == 1
+                        ? TransitionState.Adjusted
+                        : TransitionState.OK;
+                case TransitionCellCollisionPhase.Building:
+                    buildingCalls++;
+                    return buildingCalls == 1
+                        ? TransitionState.Adjusted
+                        : TransitionState.OK;
+                case TransitionCellCollisionPhase.Objects:
+                    objectCalls++;
+                    return objectCalls == 1
+                        ? TransitionState.Adjusted
+                        : TransitionState.OK;
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(phase));
+            }
+        };
+
+        Vector3 current = new(10f, 10f, 5f);
+        Vector3 target = current + new Vector3(0.05f, 0f, 0f);
+        var body = new PhysicsBody
+        {
+            Position = current,
+            Orientation = Quaternion.Identity,
+            TransientState = TransientStateFlags.Active,
+        };
+
+        ResolveResult result = engine.ResolveWithTransition(
+            current,
+            target,
+            Cell,
+            sphereRadius: 0.48f,
+            sphereHeight: 1.835f,
+            stepUpHeight: 0.6f,
+            stepDownHeight: 1.5f,
+            isOnGround: false,
+            body,
+            moverFlags: ObjectInfoState.IsPlayer | ObjectInfoState.EdgeSlide,
+            movingEntityId: 0x000F4243u);
+
+        Assert.True(result.Ok);
+        Assert.Equal(target, result.Position);
+
+        // The first inner budget of three passes stops successively at env,
+        // building, and objects. The second outer attempt starts a fourth
+        // complete pass and succeeds. A flattened N=3 loop cannot reach it.
+        TransitionCellCollisionPhase[] expected =
+        [
+            TransitionCellCollisionPhase.Environment,
+            TransitionCellCollisionPhase.Environment,
+            TransitionCellCollisionPhase.Building,
+            TransitionCellCollisionPhase.Environment,
+            TransitionCellCollisionPhase.Building,
+            TransitionCellCollisionPhase.Objects,
+            TransitionCellCollisionPhase.Environment,
+            TransitionCellCollisionPhase.Building,
+            TransitionCellCollisionPhase.Objects,
+        ];
+        Assert.Equal(expected, phases);
+        Assert.Equal(4, environmentCalls);
+        Assert.Equal(3, buildingCalls);
+        Assert.Equal(2, objectCalls);
+        Assert.True(
+            environmentCalls > 3,
+            "The fixture must require more complete cell passes than the "
+            + "outer N=3 budget while remaining inside retail's N×N budget.");
+    }
+
+    private static PhysicsEngine BuildEngine(bool preparedFlat)
+    {
+        var (root, resolved) = BSPStepUpFixtures.FlatRoof();
+        var normalized = new Dictionary<ushort, ResolvedPolygon>(resolved.Count);
+        foreach ((ushort id, ResolvedPolygon polygon) in resolved)
+        {
+            normalized.Add(id, new ResolvedPolygon
+            {
+                Id = id,
+                Vertices = polygon.Vertices,
+                Plane = polygon.Plane,
+                NumPoints = polygon.NumPoints,
+                SidesType = polygon.SidesType,
+            });
+        }
+        var physics = new GfxObjPhysics
+        {
+            SourceId = ShellGfxObj,
+            BSP = new PhysicsBSPTree { Root = root },
+            Resolved = normalized,
+            BoundingSphere = root.BoundingSphere,
+        };
+
+        var cache = new PhysicsDataCache();
+        if (preparedFlat)
+        {
+            cache.CollisionTraversalMode = CollisionTraversalMode.Flat;
+            cache.CacheGfxObj(
+                ShellGfxObj,
+                FlatCollisionAssetBuilder.FlattenGfxObj(physics));
+        }
+        else
+        {
+            cache.RegisterGfxObjForTest(ShellGfxObj, physics);
+        }
+
+        // The far-away shell makes the building channel execute its actual
+        // graph/flat traversal and return OK without affecting the mover.
+        cache.CacheBuilding(
+            Cell,
+            Array.Empty<BldPortalInfo>(),
+            Matrix4x4.CreateTranslation(100f, 100f, 100f),
+            ShellGfxObj);
+
+        var engine = new PhysicsEngine { DataCache = cache };
+        var heights = new byte[81];
+        var heightTable = new float[256];
+        Array.Fill(heightTable, -1000f);
+        engine.AddLandblock(
+            Landblock,
+            new TerrainSurface(heights, heightTable),
+            Array.Empty<CellSurface>(),
+            Array.Empty<PortalPlane>(),
+            0f,
+            0f);
+        return engine;
+    }
+}