From ddae5704da98bb532041d64cb9fc021c949163d6 Mon Sep 17 00:00:00 2001 From: Erik Date: Thu, 3 Sep 2026 22:52:33 +0200 Subject: [PATCH] =?UTF-8?q?tools(ui-probe):=20mouselook=20=20=20ve?= =?UTF-8?q?rb=20=E2=80=94=20raw=20mouse-look=20delta=20injection=20for=20s?= =?UTF-8?q?cripted=20camera=20pitch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #464 needs a repeatable tilted self-gate frame at the cathedral stair-arch pose, but the automation's `drag at`/`mousemove` verbs only move the retained-UI cursor — they never reach mouse-look, so every scripted route replays the DEFAULT camera and can't reproduce the owner's upward-tilted frame (docs/ISSUES.md #464, the 2026-09-03 22:07 transcript note: "the automation's `drag at` verb does not drive mouse-look"). Add `mouselook `: it calls the SAME GameplayInputFrameController.QueueRawMouseDelta the real mouse's Silk MouseMove callback drives (CameraPointerInputController. ProcessMouseMove), through an injected delegate threaded RetailUiAutomationScriptRunner <- RetailUiProbeBindings <- InteractionRetainedUiDependencies.GameplayInputFrame. That last seam is resolved fresh on every call rather than captured once at mount, since GameplayInputFrameController is created per live session (SessionPlayerComposition), strictly after the retained UI composes and across reconnects — the same never-capture-a-deferred-Func discipline the secure-trade command-bus regression taught (claude-memory/feedback_resolve_deferred_funcs_per_call.md). The delta only takes effect while mouse-look is active (bracket with `input down`/`up CameraInstantMouseLook`) and one call consumes exactly one raw sample on the controller's next tick, so a route must `sleep` between calls — documented on DoMouseLook and the class doc comment. Co-Authored-By: Claude Fable 5.1 --- .../InteractionRetainedUiComposition.cs | 18 +++- src/AcDream.App/Rendering/GameWindow.cs | 8 +- src/AcDream.App/UI/RetailUiRuntime.cs | 11 ++- .../Testing/RetailUiAutomationScriptRunner.cs | 43 ++++++++- .../UI/RetailUiAutomationProbeTests.cs | 92 +++++++++++++++++++ 5 files changed, 165 insertions(+), 7 deletions(-) diff --git a/src/AcDream.App/Composition/InteractionRetainedUiComposition.cs b/src/AcDream.App/Composition/InteractionRetainedUiComposition.cs index af8a6608..01763f22 100644 --- a/src/AcDream.App/Composition/InteractionRetainedUiComposition.cs +++ b/src/AcDream.App/Composition/InteractionRetainedUiComposition.cs @@ -97,7 +97,14 @@ internal sealed record InteractionRetainedUiDependencies( Func? RenderPackDiagnostics = null, string? ScreenshotsDirectory = null, - AppAutomationSurface? Automation = null) + AppAutomationSurface? Automation = null, + // Issue #464: the live gameplay-frame owner's raw mouse-look queue is + // deferred exactly like `late.*` above — GameplayInputFrameController + // is created per session (SessionPlayerComposition), strictly after + // this composition mounts, and is torn down/re-created across + // reconnects. Resolve it INSIDE the lambda on every call, never capture + // the result once (see feedback_resolve_deferred_funcs_per_call). + Func? GameplayInputFrame = null) { public RuntimeActionState Actions => Runtime.ActionOwner; @@ -1145,7 +1152,14 @@ internal sealed class RetailInteractionRetainedUiCompositionFactory action => d.InputDispatcher?.TryInvokeAutomationAction(action) == true, (action, held) => d.InputDispatcher?.TrySetAutomationActionHeld(action, held) == true, - late.Automation), + late.Automation, + // Issue #464: same live controller + method the real + // mouse's Silk callback drives + // (CameraPointerInputController.ProcessMouseMove -> + // _gameplayFrame.QueueRawMouseDelta) — resolved fresh on + // every call per d.GameplayInputFrame's own doc. + QueueMouseLookDelta: (dx, dy) => + d.GameplayInputFrame?.Invoke()?.QueueRawMouseDelta(dx, dy)), Keyboard: new KeyboardRuntimeBindings( d.InputDispatcher, d.KeyBindingsFilePath), diff --git a/src/AcDream.App/Rendering/GameWindow.cs b/src/AcDream.App/Rendering/GameWindow.cs index a3f8e1c8..ad9e8a08 100644 --- a/src/AcDream.App/Rendering/GameWindow.cs +++ b/src/AcDream.App/Rendering/GameWindow.cs @@ -1577,7 +1577,13 @@ public sealed class GameWindow : settingsDevTools.RenderPacks, _renderPackDiagnostics.CaptureDiagnostics, _applicationPaths.ScreenshotsDirectory, - _automation), + _automation, + // Issue #464: resolved fresh per call (see the + // dependency's own doc comment) — _gameplayInputFrame is + // null until SessionPlayerComposition mounts and stays + // whatever it was set to across reconnects, matching the + // real mouse's own path to the same controller. + GameplayInputFrame: () => _gameplayInputFrame), _retailUiLease, this).Compose( platformResult, diff --git a/src/AcDream.App/UI/RetailUiRuntime.cs b/src/AcDream.App/UI/RetailUiRuntime.cs index 384b7f27..0bd116d1 100644 --- a/src/AcDream.App/UI/RetailUiRuntime.cs +++ b/src/AcDream.App/UI/RetailUiRuntime.cs @@ -392,7 +392,13 @@ public sealed record RetailUiProbeBindings( Action Log, Func PressInput, Func SetInputHeld, - Testing.IRetailUiAutomationRuntime? Runtime = null); + Testing.IRetailUiAutomationRuntime? Runtime = null, + // Issue #464: injects a raw mouse-look delta exactly the way the real + // mouse's Silk callback does (GameplayInputFrameController. + // QueueRawMouseDelta), so a scripted route can pitch the chase camera. + // Null (the default) leaves the runner's `mouselook` verb reporting + // "unavailable" rather than silently no-opping. + Action? QueueMouseLookDelta = null); public sealed record RetailUiCursorBindings( CursorFeedbackController Feedback, @@ -693,7 +699,8 @@ public sealed class RetailUiRuntime : IDisposable ChatChannelKind.Say), bindings.Probe.PressInput, bindings.Probe.SetInputHeld, - bindings.Probe.Runtime); + bindings.Probe.Runtime, + bindings.Probe.QueueMouseLookDelta); } } diff --git a/src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs b/src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs index dc9012fa..c7188992 100644 --- a/src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs +++ b/src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs @@ -114,7 +114,9 @@ public interface IRetailUiAutomationRuntime /// Tiny one-thread script runner for . /// It is intended for local diagnostic launches, not for gameplay. Commands /// execute on render ticks. Pointer commands use the same -/// path as physical mouse input; semantic input uses the production dispatcher. +/// path as physical mouse input; semantic input uses the production dispatcher; +/// mouselook injects a raw mouse-look delta the same way the real +/// mouse's Silk callback does (see ). /// public sealed class RetailUiAutomationScriptRunner : IDisposable { @@ -124,6 +126,7 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable private readonly Func? _pressInput; private readonly Func? _setInputHeld; private readonly IRetailUiAutomationRuntime? _runtime; + private readonly Action? _queueMouseLookDelta; private readonly List _commands = new(); private readonly HashSet _heldInputs = new(); private readonly bool _dumpOnStart; @@ -146,7 +149,8 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable Action? submitCommand = null, Func? pressInput = null, Func? setInputHeld = null, - IRetailUiAutomationRuntime? runtime = null) + IRetailUiAutomationRuntime? runtime = null, + Action? queueMouseLookDelta = null) { _probe = probe ?? throw new ArgumentNullException(nameof(probe)); _log = log ?? (_ => { }); @@ -154,6 +158,7 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable _pressInput = pressInput; _setInputHeld = setInputHeld; _runtime = runtime; + _queueMouseLookDelta = queueMouseLookDelta; _dumpOnStart = dumpOnStart; if (!string.IsNullOrWhiteSpace(scriptPath)) @@ -272,6 +277,7 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable "assert" => DoAssert(command), "command" => DoCommand(command), "input" => DoInput(command), + "mouselook" => DoMouseLook(command), "checkpoint" => DoCheckpoint(command), "renderpack" => DoRenderPack(command), "resize" => DoResize(command), @@ -729,6 +735,36 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable } } + /// `mouselook <dx> <dy>` — injects one raw mouse-look + /// delta sample the exact way the real mouse does: retail's Silk + /// IMouse.MouseMove callback resolves a frame delta and hands it to + /// + /// (see CameraPointerInputController.ProcessMouseMove), and this + /// verb calls the SAME method on the SAME live controller through the + /// injected delegate — it is not a synthetic UI-cursor move like + /// mousemove/drag at, which only move the retained-UI + /// pointer and cannot pitch the camera (issue #464). Two things a route + /// must get right: (1) the delta only takes effect while mouse-look is + /// ACTIVE, so bracket the call with input down + /// CameraInstantMouseLook before and input up + /// CameraInstantMouseLook after; (2) one call queues exactly one raw + /// sample, consumed on the controller's next frame tick, so a route + /// issuing several deltas must separate them with sleep — calling + /// this back to back overwrites the pending sample instead of + /// accumulating it. + private bool DoMouseLook(ScriptCommand command) + { + var p = command.Parts; + if (p.Length != 3 + || !TryParseFloat(p[1], out float dx) + || !TryParseFloat(p[2], out float dy)) + return Stop(command, "usage: mouselook "); + if (_queueMouseLookDelta is null) + return Stop(command, "mouselook injection is unavailable"); + _queueMouseLookDelta(dx, dy); + return true; + } + private bool DoCheckpoint(ScriptCommand command) { if (command.Parts.Length != 2) @@ -932,5 +968,8 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable private static bool TryParseInt(string value, out int parsed) => int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out parsed); + private static bool TryParseFloat(string value, out float parsed) + => float.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out parsed); + private readonly record struct ScriptCommand(int LineNumber, string Text, string[] Parts); } diff --git a/tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs b/tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs index 68de918a..b3b43d0d 100644 --- a/tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs +++ b/tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs @@ -626,6 +626,98 @@ public sealed class RetailUiAutomationProbeTests } } + [Fact] + public void ScriptRunner_mouselook_injectsRawDeltaThroughDelegate() + { + var (root, _, _, _, objects) = RootWithTwoItemLists(); + var probe = new RetailUiAutomationProbe(root, objects); + var logs = new List(); + var deltas = new List<(float Dx, float Dy)>(); + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ui-probe.txt"); + File.WriteAllLines(path, ["mouselook 3 -4.5"]); + + try + { + using var runner = new RetailUiAutomationScriptRunner( + probe, + path, + dumpOnStart: false, + log: logs.Add, + queueMouseLookDelta: (dx, dy) => deltas.Add((dx, dy))); + + runner.Tick(0d); + + Assert.True(runner.Completed); + Assert.Contains(logs, line => line.Contains("UI probe script complete")); + Assert.Equal([(3f, -4.5f)], deltas); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void ScriptRunner_mouselook_wrongArity_stopsWithUsageAndNeverInvokesDelegate() + { + var (root, _, _, _, objects) = RootWithTwoItemLists(); + var probe = new RetailUiAutomationProbe(root, objects); + var logs = new List(); + var deltas = new List<(float Dx, float Dy)>(); + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ui-probe.txt"); + File.WriteAllLines(path, ["mouselook 1"]); + + try + { + using var runner = new RetailUiAutomationScriptRunner( + probe, + path, + dumpOnStart: false, + log: logs.Add, + queueMouseLookDelta: (dx, dy) => deltas.Add((dx, dy))); + + runner.Tick(0d); + + Assert.True(runner.Completed); + Assert.Contains(logs, line => + line.Contains("usage: mouselook ", StringComparison.Ordinal)); + Assert.Empty(deltas); + } + finally + { + File.Delete(path); + } + } + + [Fact] + public void ScriptRunner_mouselook_noDelegate_stopsAsUnavailable() + { + var (root, _, _, _, objects) = RootWithTwoItemLists(); + var probe = new RetailUiAutomationProbe(root, objects); + var logs = new List(); + string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ui-probe.txt"); + File.WriteAllLines(path, ["mouselook 3 -4.5"]); + + try + { + using var runner = new RetailUiAutomationScriptRunner( + probe, + path, + dumpOnStart: false, + log: logs.Add); + + runner.Tick(0d); + + Assert.True(runner.Completed); + Assert.Contains(logs, line => + line.Contains("mouselook injection is unavailable", StringComparison.Ordinal)); + } + finally + { + File.Delete(path); + } + } + [Fact] public void ScriptRunner_dispose_releasesInjectedHeldActions() {