perf(headless): establish K4 resource telemetry
This commit is contained in:
Erik
parent
827a039760
commit
cb512fd091
5 changed files with 662 additions and 4 deletions
|
|
@ -13,6 +13,7 @@ internal sealed class HeadlessProcessHost : IDisposable
|
|||
private readonly HeadlessProcessScheduler _scheduler;
|
||||
private readonly HeadlessDiagnosticWriter _diagnostics;
|
||||
private readonly HeadlessProcessContentOwner? _content;
|
||||
private readonly HeadlessProcessResourceSampler _resources;
|
||||
private int _disposeIndex;
|
||||
private bool _disposed;
|
||||
|
||||
|
|
@ -50,6 +51,7 @@ internal sealed class HeadlessProcessHost : IDisposable
|
|||
var sessions = new List<HeadlessSessionHost>(
|
||||
configuration.Sessions.Count);
|
||||
HeadlessProcessContentOwner? content = null;
|
||||
HeadlessProcessResourceSampler? resources = null;
|
||||
try
|
||||
{
|
||||
if (configuration.Process?.Content is { } contentDescriptor)
|
||||
|
|
@ -102,14 +104,23 @@ internal sealed class HeadlessProcessHost : IDisposable
|
|||
}
|
||||
|
||||
_sessions = sessions.ToArray();
|
||||
resources = new HeadlessProcessResourceSampler(timeProvider);
|
||||
_scheduler = new HeadlessProcessScheduler(
|
||||
_sessions,
|
||||
timeProvider);
|
||||
timeProvider,
|
||||
observation: snapshot =>
|
||||
CaptureResources(
|
||||
"periodic",
|
||||
resources,
|
||||
snapshot,
|
||||
content));
|
||||
_resources = resources;
|
||||
_content = content;
|
||||
_disposeIndex = _sessions.Length - 1;
|
||||
}
|
||||
catch
|
||||
{
|
||||
resources?.Dispose();
|
||||
for (int index = sessions.Count - 1; index >= 0; index--)
|
||||
sessions[index].Dispose();
|
||||
content?.Dispose();
|
||||
|
|
@ -164,6 +175,12 @@ internal sealed class HeadlessProcessHost : IDisposable
|
|||
"running",
|
||||
session.Runtime);
|
||||
}
|
||||
_scheduler.RebaseDeadlinesAfterSessionStart();
|
||||
CaptureResources(
|
||||
"running-start",
|
||||
_resources,
|
||||
_scheduler.CaptureSnapshot(),
|
||||
_content);
|
||||
|
||||
try
|
||||
{
|
||||
|
|
@ -180,6 +197,11 @@ internal sealed class HeadlessProcessHost : IDisposable
|
|||
return HeadlessExitCode.RuntimeError;
|
||||
}
|
||||
|
||||
CaptureResources(
|
||||
"running-stop",
|
||||
_resources,
|
||||
_scheduler.CaptureSnapshot(),
|
||||
_content);
|
||||
return _scheduler.CaptureSnapshot().FaultedSessionCount > 0
|
||||
? HeadlessExitCode.RuntimeError
|
||||
: HeadlessExitCode.Success;
|
||||
|
|
@ -195,6 +217,25 @@ internal sealed class HeadlessProcessHost : IDisposable
|
|||
_disposeIndex--;
|
||||
}
|
||||
_content?.Dispose();
|
||||
CaptureResources(
|
||||
"disposed",
|
||||
_resources,
|
||||
_scheduler.CaptureSnapshot(),
|
||||
_content);
|
||||
_resources.Dispose();
|
||||
_disposed = true;
|
||||
}
|
||||
|
||||
private void CaptureResources(
|
||||
string state,
|
||||
HeadlessProcessResourceSampler resources,
|
||||
HeadlessSchedulerSnapshot scheduler,
|
||||
HeadlessProcessContentOwner? content)
|
||||
{
|
||||
HeadlessProcessResourceSnapshot snapshot = resources.Capture(
|
||||
_sessions,
|
||||
scheduler,
|
||||
content?.CaptureSnapshot());
|
||||
_diagnostics.Resources(state, snapshot);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@ namespace AcDream.Headless.Hosting;
|
|||
|
||||
internal sealed class HeadlessProcessScheduler
|
||||
{
|
||||
private static readonly TimeSpan MinimumTimerDelay =
|
||||
TimeSpan.FromMilliseconds(1);
|
||||
|
||||
private sealed class Slot
|
||||
{
|
||||
internal Slot(
|
||||
|
|
@ -25,15 +28,23 @@ internal sealed class HeadlessProcessScheduler
|
|||
private readonly long _periodTicks;
|
||||
private readonly int _maximumCatchUpTurns;
|
||||
private readonly Slot[] _slots;
|
||||
private readonly Action<HeadlessSchedulerSnapshot>? _observation;
|
||||
private readonly long _observationPeriodTicks;
|
||||
private long _nextObservationDeadline;
|
||||
private long _waitCount;
|
||||
private long _turnCount;
|
||||
private long _catchUpCollapseCount;
|
||||
private long _lateDeadlineCount;
|
||||
private long _totalLatenessTicks;
|
||||
private long _maximumLatenessTicks;
|
||||
|
||||
internal HeadlessProcessScheduler(
|
||||
IReadOnlyList<HeadlessSessionHost> sessions,
|
||||
TimeProvider? timeProvider = null,
|
||||
TimeSpan? turnPeriod = null,
|
||||
int maximumCatchUpTurns = 8)
|
||||
int maximumCatchUpTurns = 8,
|
||||
Action<HeadlessSchedulerSnapshot>? observation = null,
|
||||
TimeSpan? observationPeriod = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(sessions);
|
||||
if (sessions.Count == 0)
|
||||
|
|
@ -57,8 +68,22 @@ internal sealed class HeadlessProcessScheduler
|
|||
_timeProvider,
|
||||
configuredTurnPeriod);
|
||||
_maximumCatchUpTurns = maximumCatchUpTurns;
|
||||
_observation = observation;
|
||||
TimeSpan configuredObservationPeriod =
|
||||
observationPeriod ?? TimeSpan.FromSeconds(30);
|
||||
if (configuredObservationPeriod <= TimeSpan.Zero)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(
|
||||
nameof(observationPeriod));
|
||||
}
|
||||
_observationPeriodTicks = DurationToTimestampTicks(
|
||||
_timeProvider,
|
||||
configuredObservationPeriod);
|
||||
|
||||
long start = _timeProvider.GetTimestamp();
|
||||
_nextObservationDeadline = AddTicks(
|
||||
start,
|
||||
_observationPeriodTicks);
|
||||
_slots = new Slot[sessions.Count];
|
||||
for (int index = 0; index < sessions.Count; index++)
|
||||
{
|
||||
|
|
@ -89,24 +114,60 @@ internal sealed class HeadlessProcessScheduler
|
|||
Interlocked.Read(ref _waitCount),
|
||||
Interlocked.Read(ref _turnCount),
|
||||
Interlocked.Read(ref _catchUpCollapseCount),
|
||||
Interlocked.Read(ref _lateDeadlineCount),
|
||||
Interlocked.Read(ref _totalLatenessTicks),
|
||||
Interlocked.Read(ref _maximumLatenessTicks),
|
||||
_timeProvider.TimestampFrequency,
|
||||
activeSessionCount,
|
||||
faultedSessionCount,
|
||||
NextDeadline());
|
||||
}
|
||||
|
||||
internal void RebaseDeadlinesAfterSessionStart()
|
||||
{
|
||||
if (Interlocked.Read(ref _waitCount) != 0L
|
||||
|| Interlocked.Read(ref _turnCount) != 0L
|
||||
|| Interlocked.Read(ref _catchUpCollapseCount) != 0L
|
||||
|| Interlocked.Read(ref _lateDeadlineCount) != 0L)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"A running headless scheduler cannot rebase its deadlines.");
|
||||
}
|
||||
|
||||
long start = _timeProvider.GetTimestamp();
|
||||
for (int index = 0; index < _slots.Length; index++)
|
||||
{
|
||||
_slots[index].LastTurnTimestamp = start;
|
||||
_slots[index].NextDeadline = AddTicks(
|
||||
start,
|
||||
_periodTicks);
|
||||
}
|
||||
_nextObservationDeadline = AddTicks(
|
||||
start,
|
||||
_observationPeriodTicks);
|
||||
}
|
||||
|
||||
internal async Task RunAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
while (!cancellationToken.IsCancellationRequested
|
||||
&& HasActiveSession())
|
||||
{
|
||||
long now = _timeProvider.GetTimestamp();
|
||||
if (DispatchDue(now))
|
||||
bool dispatched = DispatchDue(now);
|
||||
bool observed = DispatchObservationDue(now);
|
||||
if (dispatched || observed)
|
||||
continue;
|
||||
|
||||
long deadline = NextDeadline();
|
||||
if (_observation is not null
|
||||
&& _nextObservationDeadline < deadline)
|
||||
{
|
||||
deadline = _nextObservationDeadline;
|
||||
}
|
||||
TimeSpan delay = deadline <= now
|
||||
? TimeSpan.Zero
|
||||
: _timeProvider.GetElapsedTime(now, deadline);
|
||||
delay = NormalizeTimerDelay(delay);
|
||||
Interlocked.Increment(ref _waitCount);
|
||||
await Task.Delay(
|
||||
delay,
|
||||
|
|
@ -116,6 +177,12 @@ internal sealed class HeadlessProcessScheduler
|
|||
}
|
||||
}
|
||||
|
||||
internal static TimeSpan NormalizeTimerDelay(TimeSpan delay) =>
|
||||
delay > TimeSpan.Zero
|
||||
&& delay < MinimumTimerDelay
|
||||
? MinimumTimerDelay
|
||||
: delay;
|
||||
|
||||
internal bool DispatchDue(long nowTimestamp)
|
||||
{
|
||||
bool dispatched = false;
|
||||
|
|
@ -132,6 +199,9 @@ internal sealed class HeadlessProcessScheduler
|
|||
if (nowTimestamp < session.ReconnectDeadline)
|
||||
continue;
|
||||
|
||||
RecordLateness(
|
||||
nowTimestamp,
|
||||
session.ReconnectDeadline);
|
||||
RuntimeSessionStartResult result =
|
||||
session.CompletePendingReconnect(nowTimestamp);
|
||||
if (result.Status
|
||||
|
|
@ -164,8 +234,32 @@ internal sealed class HeadlessProcessScheduler
|
|||
return dispatched;
|
||||
}
|
||||
|
||||
internal bool DispatchObservationDue(long nowTimestamp)
|
||||
{
|
||||
if (_observation is null
|
||||
|| nowTimestamp < _nextObservationDeadline)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
long dueCount =
|
||||
1L
|
||||
+ ((nowTimestamp - _nextObservationDeadline)
|
||||
/ _observationPeriodTicks);
|
||||
long advance = dueCount
|
||||
> long.MaxValue / _observationPeriodTicks
|
||||
? long.MaxValue
|
||||
: dueCount * _observationPeriodTicks;
|
||||
_nextObservationDeadline = AddTicks(
|
||||
_nextObservationDeadline,
|
||||
advance);
|
||||
_observation(CaptureSnapshot());
|
||||
return true;
|
||||
}
|
||||
|
||||
private void DispatchSessionDue(Slot slot, long nowTimestamp)
|
||||
{
|
||||
RecordLateness(nowTimestamp, slot.NextDeadline);
|
||||
long dueCount =
|
||||
1L + ((nowTimestamp - slot.NextDeadline) / _periodTicks);
|
||||
int turns = (int)Math.Min(
|
||||
|
|
@ -203,6 +297,27 @@ internal sealed class HeadlessProcessScheduler
|
|||
Interlocked.Increment(ref _catchUpCollapseCount);
|
||||
}
|
||||
|
||||
private void RecordLateness(long nowTimestamp, long deadline)
|
||||
{
|
||||
if (nowTimestamp <= deadline)
|
||||
return;
|
||||
|
||||
long lateness = nowTimestamp - deadline;
|
||||
Interlocked.Increment(ref _lateDeadlineCount);
|
||||
Interlocked.Add(ref _totalLatenessTicks, lateness);
|
||||
long maximum = Interlocked.Read(ref _maximumLatenessTicks);
|
||||
while (lateness > maximum)
|
||||
{
|
||||
long observed = Interlocked.CompareExchange(
|
||||
ref _maximumLatenessTicks,
|
||||
lateness,
|
||||
maximum);
|
||||
if (observed == maximum)
|
||||
break;
|
||||
maximum = observed;
|
||||
}
|
||||
}
|
||||
|
||||
private long NextDeadline()
|
||||
{
|
||||
long next = long.MaxValue;
|
||||
|
|
@ -257,6 +372,25 @@ internal readonly record struct HeadlessSchedulerSnapshot(
|
|||
long WaitCount,
|
||||
long TurnCount,
|
||||
long CatchUpCollapseCount,
|
||||
long LateDeadlineCount,
|
||||
long TotalLatenessTicks,
|
||||
long MaximumLatenessTicks,
|
||||
long TimestampFrequency,
|
||||
int ActiveSessionCount,
|
||||
int FaultedSessionCount,
|
||||
long NextDeadline);
|
||||
long NextDeadline)
|
||||
{
|
||||
internal double MeanLatenessMilliseconds =>
|
||||
LateDeadlineCount == 0
|
||||
? 0d
|
||||
: TimestampTicksToMilliseconds(
|
||||
TotalLatenessTicks / (double)LateDeadlineCount);
|
||||
|
||||
internal double MaximumLatenessMilliseconds =>
|
||||
TimestampTicksToMilliseconds(MaximumLatenessTicks);
|
||||
|
||||
private double TimestampTicksToMilliseconds(double ticks) =>
|
||||
TimestampFrequency <= 0L
|
||||
? 0d
|
||||
: ticks * 1000d / TimestampFrequency;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue