erik/acdream
1
0
Fork 0
Code Issues Pull requests Projects Releases 6 Packages Wiki Activity Actions

tools(ui-probe): mouselook <dx> <dy> verb — raw mouse-look delta injection for scripted camera pitch

Browse source 
Issue #464 needs a repeatable tilted self-gate frame at the cathedral
stair-arch pose, but the automation's `drag at`/`mousemove` verbs only
move the retained-UI cursor — they never reach mouse-look, so every
scripted route replays the DEFAULT camera and can't reproduce the
owner's upward-tilted frame (docs/ISSUES.md #464, the 2026-09-03
22:07 transcript note: "the automation's `drag at` verb does not
drive mouse-look").

Add `mouselook <dx> <dy>`: it calls the SAME
GameplayInputFrameController.QueueRawMouseDelta the real mouse's Silk
MouseMove callback drives (CameraPointerInputController.
ProcessMouseMove), through an injected delegate threaded
RetailUiAutomationScriptRunner <- RetailUiProbeBindings <-
InteractionRetainedUiDependencies.GameplayInputFrame. That last seam
is resolved fresh on every call rather than captured once at mount,
since GameplayInputFrameController is created per live session
(SessionPlayerComposition), strictly after the retained UI composes
and across reconnects — the same never-capture-a-deferred-Func
discipline the secure-trade command-bus regression taught
(claude-memory/feedback_resolve_deferred_funcs_per_call.md).

The delta only takes effect while mouse-look is active (bracket with
`input down`/`up CameraInstantMouseLook`) and one call consumes
exactly one raw sample on the controller's next tick, so a route must
`sleep` between calls — documented on DoMouseLook and the class doc
comment.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Erik 2026-09-03 22:52:33 +02:00
parent 37d1357baa
commit ddae5704da
5 changed files with 165 additions and 7 deletions
Download patch file Download diff file Expand all files Collapse all files

18
src/AcDream.App/Composition/InteractionRetainedUiComposition.cs
View file

@ -97,7 +97,14 @@ internal sealed record InteractionRetainedUiDependencies(
Func<AcDream.App.Rendering.Packs.RenderPackDiagnosticsSnapshot>?
RenderPackDiagnostics = null,
string? ScreenshotsDirectory = null,
AppAutomationSurface? Automation = null)
AppAutomationSurface? Automation = null,
// Issue #464: the live gameplay-frame owner's raw mouse-look queue is
// deferred exactly like `late.*` above — GameplayInputFrameController
// is created per session (SessionPlayerComposition), strictly after
// this composition mounts, and is torn down/re-created across
// reconnects. Resolve it INSIDE the lambda on every call, never capture
// the result once (see feedback_resolve_deferred_funcs_per_call).
Func<GameplayInputFrameController?>? GameplayInputFrame = null)
{
public RuntimeActionState Actions => Runtime.ActionOwner;
@ -1145,7 +1152,14 @@ internal sealed class RetailInteractionRetainedUiCompositionFactory
action => d.InputDispatcher?.TryInvokeAutomationAction(action) == true,
(action, held) =>
d.InputDispatcher?.TrySetAutomationActionHeld(action, held) == true,
late.Automation),
late.Automation,
// Issue #464: same live controller + method the real
// mouse's Silk callback drives
// (CameraPointerInputController.ProcessMouseMove ->
// _gameplayFrame.QueueRawMouseDelta) — resolved fresh on
// every call per d.GameplayInputFrame's own doc.
QueueMouseLookDelta: (dx, dy) =>
d.GameplayInputFrame?.Invoke()?.QueueRawMouseDelta(dx, dy)),
Keyboard: new KeyboardRuntimeBindings(
d.InputDispatcher,
d.KeyBindingsFilePath),

8
src/AcDream.App/Rendering/GameWindow.cs
View file

@ -1577,7 +1577,13 @@ public sealed class GameWindow :
settingsDevTools.RenderPacks,
_renderPackDiagnostics.CaptureDiagnostics,
_applicationPaths.ScreenshotsDirectory,
_automation),
_automation,
// Issue #464: resolved fresh per call (see the
// dependency's own doc comment) — _gameplayInputFrame is
// null until SessionPlayerComposition mounts and stays
// whatever it was set to across reconnects, matching the
// real mouse's own path to the same controller.
GameplayInputFrame: () => _gameplayInputFrame),
_retailUiLease,
this).Compose(
platformResult,

11
src/AcDream.App/UI/RetailUiRuntime.cs
View file

@ -392,7 +392,13 @@ public sealed record RetailUiProbeBindings(
Action<string> Log,
Func<InputAction, bool> PressInput,
Func<InputAction, bool, bool> SetInputHeld,
Testing.IRetailUiAutomationRuntime? Runtime = null);
Testing.IRetailUiAutomationRuntime? Runtime = null,
// Issue #464: injects a raw mouse-look delta exactly the way the real
// mouse's Silk callback does (GameplayInputFrameController.
// QueueRawMouseDelta), so a scripted route can pitch the chase camera.
// Null (the default) leaves the runner's `mouselook` verb reporting
// "unavailable" rather than silently no-opping.
Action<float, float>? QueueMouseLookDelta = null);
public sealed record RetailUiCursorBindings(
CursorFeedbackController Feedback,
@ -693,7 +699,8 @@ public sealed class RetailUiRuntime : IDisposable
ChatChannelKind.Say),
bindings.Probe.PressInput,
bindings.Probe.SetInputHeld,
bindings.Probe.Runtime);
bindings.Probe.Runtime,
bindings.Probe.QueueMouseLookDelta);
}
}

43
src/AcDream.App/UI/Testing/RetailUiAutomationScriptRunner.cs
View file

@ -114,7 +114,9 @@ public interface IRetailUiAutomationRuntime
/// Tiny one-thread script runner for <see cref="RetailUiAutomationProbe"/>.
/// It is intended for local diagnostic launches, not for gameplay. Commands
/// execute on render ticks. Pointer commands use the same <see cref="UiRoot"/>
/// path as physical mouse input; semantic input uses the production dispatcher.
/// path as physical mouse input; semantic input uses the production dispatcher;
/// <c>mouselook</c> injects a raw mouse-look delta the same way the real
/// mouse's Silk callback does (see <see cref="DoMouseLook"/>).
/// </summary>
public sealed class RetailUiAutomationScriptRunner : IDisposable
{
@ -124,6 +126,7 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
private readonly Func<InputAction, bool>? _pressInput;
private readonly Func<InputAction, bool, bool>? _setInputHeld;
private readonly IRetailUiAutomationRuntime? _runtime;
private readonly Action<float, float>? _queueMouseLookDelta;
private readonly List<ScriptCommand> _commands = new();
private readonly HashSet<InputAction> _heldInputs = new();
private readonly bool _dumpOnStart;
@ -146,7 +149,8 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
Action<string>? submitCommand = null,
Func<InputAction, bool>? pressInput = null,
Func<InputAction, bool, bool>? setInputHeld = null,
IRetailUiAutomationRuntime? runtime = null)
IRetailUiAutomationRuntime? runtime = null,
Action<float, float>? queueMouseLookDelta = null)
{
_probe = probe ?? throw new ArgumentNullException(nameof(probe));
_log = log ?? (_ => { });
@ -154,6 +158,7 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
_pressInput = pressInput;
_setInputHeld = setInputHeld;
_runtime = runtime;
_queueMouseLookDelta = queueMouseLookDelta;
_dumpOnStart = dumpOnStart;
if (!string.IsNullOrWhiteSpace(scriptPath))
@ -272,6 +277,7 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
"assert" => DoAssert(command),
"command" => DoCommand(command),
"input" => DoInput(command),
"mouselook" => DoMouseLook(command),
"checkpoint" => DoCheckpoint(command),
"renderpack" => DoRenderPack(command),
"resize" => DoResize(command),
@ -729,6 +735,36 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
}
}
/// <summary>`mouselook &lt;dx&gt; &lt;dy&gt;` — injects one raw mouse-look
/// delta sample the exact way the real mouse does: retail's Silk
/// <c>IMouse.MouseMove</c> callback resolves a frame delta and hands it to
/// <see cref="AcDream.App.Input.GameplayInputFrameController.QueueRawMouseDelta"/>
/// (see <c>CameraPointerInputController.ProcessMouseMove</c>), and this
/// verb calls the SAME method on the SAME live controller through the
/// injected delegate — it is not a synthetic UI-cursor move like
/// <c>mousemove</c>/<c>drag at</c>, which only move the retained-UI
/// pointer and cannot pitch the camera (issue #464). Two things a route
/// must get right: (1) the delta only takes effect while mouse-look is
/// ACTIVE, so bracket the call with <c>input down
/// CameraInstantMouseLook</c> before and <c>input up
/// CameraInstantMouseLook</c> after; (2) one call queues exactly one raw
/// sample, consumed on the controller's next frame tick, so a route
/// issuing several deltas must separate them with <c>sleep</c> — calling
/// this back to back overwrites the pending sample instead of
/// accumulating it.</summary>
private bool DoMouseLook(ScriptCommand command)
{
var p = command.Parts;
if (p.Length != 3
|| !TryParseFloat(p[1], out float dx)
|| !TryParseFloat(p[2], out float dy))
return Stop(command, "usage: mouselook <dx> <dy>");
if (_queueMouseLookDelta is null)
return Stop(command, "mouselook injection is unavailable");
_queueMouseLookDelta(dx, dy);
return true;
}
private bool DoCheckpoint(ScriptCommand command)
{
if (command.Parts.Length != 2)
@ -932,5 +968,8 @@ public sealed class RetailUiAutomationScriptRunner : IDisposable
private static bool TryParseInt(string value, out int parsed)
=> int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out parsed);
private static bool TryParseFloat(string value, out float parsed)
=> float.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out parsed);
private readonly record struct ScriptCommand(int LineNumber, string Text, string[] Parts);
}

92
tests/AcDream.App.Tests/UI/RetailUiAutomationProbeTests.cs
View file

@ -626,6 +626,98 @@ public sealed class RetailUiAutomationProbeTests
}
}
[Fact]
public void ScriptRunner_mouselook_injectsRawDeltaThroughDelegate()
{
var (root, _, _, _, objects) = RootWithTwoItemLists();
var probe = new RetailUiAutomationProbe(root, objects);
var logs = new List<string>();
var deltas = new List<(float Dx, float Dy)>();
string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ui-probe.txt");
File.WriteAllLines(path, ["mouselook 3 -4.5"]);
try
{
using var runner = new RetailUiAutomationScriptRunner(
probe,
path,
dumpOnStart: false,
log: logs.Add,
queueMouseLookDelta: (dx, dy) => deltas.Add((dx, dy)));
runner.Tick(0d);
Assert.True(runner.Completed);
Assert.Contains(logs, line => line.Contains("UI probe script complete"));
Assert.Equal([(3f, -4.5f)], deltas);
}
finally
{
File.Delete(path);
}
}
[Fact]
public void ScriptRunner_mouselook_wrongArity_stopsWithUsageAndNeverInvokesDelegate()
{
var (root, _, _, _, objects) = RootWithTwoItemLists();
var probe = new RetailUiAutomationProbe(root, objects);
var logs = new List<string>();
var deltas = new List<(float Dx, float Dy)>();
string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ui-probe.txt");
File.WriteAllLines(path, ["mouselook 1"]);
try
{
using var runner = new RetailUiAutomationScriptRunner(
probe,
path,
dumpOnStart: false,
log: logs.Add,
queueMouseLookDelta: (dx, dy) => deltas.Add((dx, dy)));
runner.Tick(0d);
Assert.True(runner.Completed);
Assert.Contains(logs, line =>
line.Contains("usage: mouselook <dx> <dy>", StringComparison.Ordinal));
Assert.Empty(deltas);
}
finally
{
File.Delete(path);
}
}
[Fact]
public void ScriptRunner_mouselook_noDelegate_stopsAsUnavailable()
{
var (root, _, _, _, objects) = RootWithTwoItemLists();
var probe = new RetailUiAutomationProbe(root, objects);
var logs = new List<string>();
string path = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName() + ".ui-probe.txt");
File.WriteAllLines(path, ["mouselook 3 -4.5"]);
try
{
using var runner = new RetailUiAutomationScriptRunner(
probe,
path,
dumpOnStart: false,
log: logs.Add);
runner.Tick(0d);
Assert.True(runner.Completed);
Assert.Contains(logs, line =>
line.Contains("mouselook injection is unavailable", StringComparison.Ordinal));
}
finally
{
File.Delete(path);
}
}
[Fact]
public void ScriptRunner_dispose_releasesInjectedHeldActions()
{