erik/acdream
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

perf(headless): establish K4 resource telemetry

Browse source
This commit is contained in:
Erik 2026-07-27 10:47:31 +02:00
parent 827a039760
commit cb512fd091
5 changed files with 662 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

42
src/AcDream.Headless/Diagnostics/HeadlessDiagnosticWriter.cs
View file

@ -1,5 +1,6 @@
using System.Diagnostics;
using System.Text.Json;
using AcDream.Headless.Hosting;
using AcDream.Runtime;
namespace AcDream.Headless.Diagnostics;
@ -72,6 +73,47 @@ internal sealed class HeadlessDiagnosticWriter
generation,
});
internal void Resources(
string state,
in HeadlessProcessResourceSnapshot snapshot)
{
ArgumentException.ThrowIfNullOrWhiteSpace(state);
HeadlessProcessContentSnapshot? content = snapshot.Content;
Write(new
{
kind = "resources",
state,
sequence = snapshot.Sequence,
monotonicTimestamp = snapshot.MonotonicTimestamp,
process = snapshot.Process,
managed = snapshot.Managed,
sessions = snapshot.Sessions,
scheduler = new
{
snapshot.Scheduler.SessionCount,
snapshot.Scheduler.WaitCount,
snapshot.Scheduler.TurnCount,
snapshot.Scheduler.CatchUpCollapseCount,
snapshot.Scheduler.LateDeadlineCount,
snapshot.Scheduler.MeanLatenessMilliseconds,
snapshot.Scheduler.MaximumLatenessMilliseconds,
snapshot.Scheduler.ActiveSessionCount,
snapshot.Scheduler.FaultedSessionCount,
snapshot.Scheduler.NextDeadline,
},
content = content is null
? null
: new
{
content.Value.LeaseCount,
content.Value.IsDisposeRequested,
content.Value.IsDisposed,
content.Value.MappedVirtualBytes,
content.Value.IsConverged,
},
});
}
private void Write<T>(T value)
{
_output.WriteLine(JsonSerializer.Serialize(value, JsonOptions));

270
src/AcDream.Headless/Diagnostics/HeadlessProcessResourceSampler.cs Normal file
View file

@ -0,0 +1,270 @@
using System.Diagnostics;
using AcDream.Headless.Hosting;
using AcDream.Runtime;
namespace AcDream.Headless.Diagnostics;
internal readonly record struct HeadlessProcessUsageSnapshot(
long WorkingSetBytes,
long PrivateBytes,
long VirtualBytes,
long TotalProcessorTimeTicks,
double SampleIntervalMilliseconds,
double CpuCorePercent,
double CpuMachinePercent,
int ProcessorCount,
int ThreadCount,
int HandleCount,
int FileDescriptorCount,
int SocketDescriptorCount);
internal readonly record struct HeadlessManagedUsageSnapshot(
long LiveBytes,
long HeapSizeBytes,
long FragmentedBytes,
long CommittedBytes,
long TotalAllocatedBytes,
long MemoryLoadBytes,
long HighMemoryLoadThresholdBytes,
double PauseTimePercentage,
int Gen0Collections,
int Gen1Collections,
int Gen2Collections);
internal readonly record struct HeadlessSessionUsageSnapshot(
int ConfiguredCount,
int InWorldCount,
int FaultedCount,
int PendingReconnectCount,
int ConvergedRuntimeCount,
int EntityCount,
int InventoryObjectCount,
int HostLeaseCount);
internal readonly record struct HeadlessProcessResourceSnapshot(
long Sequence,
long MonotonicTimestamp,
HeadlessProcessUsageSnapshot Process,
HeadlessManagedUsageSnapshot Managed,
HeadlessSessionUsageSnapshot Sessions,
HeadlessSchedulerSnapshot Scheduler,
HeadlessProcessContentSnapshot? Content);
/// <summary>
/// Captures one low-frequency process-wide resource sample. The sampler owns
/// no timer and no worker; the absolute-deadline process scheduler invokes it.
/// </summary>
internal sealed class HeadlessProcessResourceSampler : IDisposable
{
private readonly TimeProvider _timeProvider;
private readonly Process _process;
private long _sequence;
private long _previousTimestamp;
private long _previousProcessorTicks;
private bool _hasPreviousSample;
private bool _disposed;
internal HeadlessProcessResourceSampler(
TimeProvider? timeProvider = null,
Process? process = null)
{
_timeProvider = timeProvider ?? TimeProvider.System;
_process = process ?? Process.GetCurrentProcess();
}
internal HeadlessProcessResourceSnapshot Capture(
IReadOnlyList<HeadlessSessionHost> sessions,
HeadlessSchedulerSnapshot scheduler,
HeadlessProcessContentSnapshot? content)
{
ObjectDisposedException.ThrowIf(_disposed, this);
ArgumentNullException.ThrowIfNull(sessions);
long now = _timeProvider.GetTimestamp();
_process.Refresh();
long processorTicks = _process.TotalProcessorTime.Ticks;
double intervalMilliseconds = 0d;
double cpuCorePercent = 0d;
double cpuMachinePercent = 0d;
if (_hasPreviousSample && now > _previousTimestamp)
{
double intervalSeconds = _timeProvider
.GetElapsedTime(_previousTimestamp, now)
.TotalSeconds;
long processorDelta = Math.Max(
0L,
processorTicks - _previousProcessorTicks);
if (intervalSeconds > 0d)
{
intervalMilliseconds = intervalSeconds * 1000d;
cpuCorePercent =
processorDelta
* 100d
/ TimeSpan.TicksPerSecond
/ intervalSeconds;
cpuMachinePercent =
cpuCorePercent
/ Math.Max(1, Environment.ProcessorCount);
}
}
_previousTimestamp = now;
_previousProcessorTicks = processorTicks;
_hasPreviousSample = true;
int inWorldCount = 0;
int faultedCount = 0;
int pendingReconnectCount = 0;
int convergedRuntimeCount = 0;
int entityCount = 0;
int inventoryObjectCount = 0;
int hostLeaseCount = 0;
for (int index = 0; index < sessions.Count; index++)
{
HeadlessSessionHost session = sessions[index];
RuntimeStateCheckpoint checkpoint =
session.Runtime.CaptureCheckpoint();
GameRuntimeOwnershipSnapshot ownership =
session.Runtime.CaptureOwnership();
if (session.Runtime.Session.IsInWorld)
inWorldCount++;
if (session.IsFaulted)
faultedCount++;
if (session.IsReconnectPending)
pendingReconnectCount++;
if (ownership.IsConverged)
convergedRuntimeCount++;
entityCount = checked(entityCount + checkpoint.EntityCount);
inventoryObjectCount = checked(
inventoryObjectCount
+ checkpoint.InventoryObjectCount);
hostLeaseCount = checked(
hostLeaseCount + ownership.HostLeaseCount);
}
GCMemoryInfo memory = GC.GetGCMemoryInfo(GCKind.Any);
CaptureLinuxDescriptors(
out int fileDescriptorCount,
out int socketDescriptorCount);
var process = new HeadlessProcessUsageSnapshot(
_process.WorkingSet64,
_process.PrivateMemorySize64,
_process.VirtualMemorySize64,
processorTicks,
intervalMilliseconds,
cpuCorePercent,
cpuMachinePercent,
Environment.ProcessorCount,
TryGetThreadCount(_process),
TryGetHandleCount(_process),
fileDescriptorCount,
socketDescriptorCount);
var managed = new HeadlessManagedUsageSnapshot(
GC.GetTotalMemory(forceFullCollection: false),
memory.HeapSizeBytes,
memory.FragmentedBytes,
memory.TotalCommittedBytes,
GC.GetTotalAllocatedBytes(precise: false),
memory.MemoryLoadBytes,
memory.HighMemoryLoadThresholdBytes,
memory.PauseTimePercentage,
GC.CollectionCount(0),
GC.CollectionCount(1),
GC.CollectionCount(2));
var sessionUsage = new HeadlessSessionUsageSnapshot(
sessions.Count,
inWorldCount,
faultedCount,
pendingReconnectCount,
convergedRuntimeCount,
entityCount,
inventoryObjectCount,
hostLeaseCount);
return new HeadlessProcessResourceSnapshot(
checked(++_sequence),
now,
process,
managed,
sessionUsage,
scheduler,
content);
}
public void Dispose()
{
if (_disposed)
return;
_process.Dispose();
_disposed = true;
}
private static int TryGetThreadCount(Process process)
{
try
{
return process.Threads.Count;
}
catch (PlatformNotSupportedException)
{
return -1;
}
catch (InvalidOperationException)
{
return -1;
}
}
private static int TryGetHandleCount(Process process)
{
try
{
return process.HandleCount;
}
catch (PlatformNotSupportedException)
{
return -1;
}
catch (InvalidOperationException)
{
return -1;
}
}
private static void CaptureLinuxDescriptors(
out int fileDescriptorCount,
out int socketDescriptorCount)
{
fileDescriptorCount = -1;
socketDescriptorCount = -1;
if (!OperatingSystem.IsLinux())
return;
try
{
int descriptors = 0;
int sockets = 0;
foreach (string path
in Directory.EnumerateFileSystemEntries("/proc/self/fd"))
{
descriptors++;
string? target = new FileInfo(path).LinkTarget;
if (target?.StartsWith(
"socket:[",
StringComparison.Ordinal) == true)
{
sockets++;
}
}
fileDescriptorCount = descriptors;
socketDescriptorCount = sockets;
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
catch (PlatformNotSupportedException)
{
}
}
}

43
src/AcDream.Headless/Hosting/HeadlessProcessHost.cs
View file

@ -13,6 +13,7 @@ internal sealed class HeadlessProcessHost : IDisposable
private readonly HeadlessProcessScheduler _scheduler;
private readonly HeadlessDiagnosticWriter _diagnostics;
private readonly HeadlessProcessContentOwner? _content;
private readonly HeadlessProcessResourceSampler _resources;
private int _disposeIndex;
private bool _disposed;
@ -50,6 +51,7 @@ internal sealed class HeadlessProcessHost : IDisposable
var sessions = new List<HeadlessSessionHost>(
configuration.Sessions.Count);
HeadlessProcessContentOwner? content = null;
HeadlessProcessResourceSampler? resources = null;
try
{
if (configuration.Process?.Content is { } contentDescriptor)
@ -102,14 +104,23 @@ internal sealed class HeadlessProcessHost : IDisposable
}
_sessions = sessions.ToArray();
resources = new HeadlessProcessResourceSampler(timeProvider);
_scheduler = new HeadlessProcessScheduler(
_sessions,
timeProvider);
timeProvider,
observation: snapshot =>
CaptureResources(
"periodic",
resources,
snapshot,
content));
_resources = resources;
_content = content;
_disposeIndex = _sessions.Length - 1;
}
catch
{
resources?.Dispose();
for (int index = sessions.Count - 1; index >= 0; index--)
sessions[index].Dispose();
content?.Dispose();
@ -164,6 +175,12 @@ internal sealed class HeadlessProcessHost : IDisposable
"running",
session.Runtime);
}
_scheduler.RebaseDeadlinesAfterSessionStart();
CaptureResources(
"running-start",
_resources,
_scheduler.CaptureSnapshot(),
_content);
try
{
@ -180,6 +197,11 @@ internal sealed class HeadlessProcessHost : IDisposable
return HeadlessExitCode.RuntimeError;
}
CaptureResources(
"running-stop",
_resources,
_scheduler.CaptureSnapshot(),
_content);
return _scheduler.CaptureSnapshot().FaultedSessionCount > 0
? HeadlessExitCode.RuntimeError
: HeadlessExitCode.Success;
@ -195,6 +217,25 @@ internal sealed class HeadlessProcessHost : IDisposable
_disposeIndex--;
}
_content?.Dispose();
CaptureResources(
"disposed",
_resources,
_scheduler.CaptureSnapshot(),
_content);
_resources.Dispose();
_disposed = true;
}
private void CaptureResources(
string state,
HeadlessProcessResourceSampler resources,
HeadlessSchedulerSnapshot scheduler,
HeadlessProcessContentOwner? content)
{
HeadlessProcessResourceSnapshot snapshot = resources.Capture(
_sessions,
scheduler,
content?.CaptureSnapshot());
_diagnostics.Resources(state, snapshot);
}
}

140
src/AcDream.Headless/Hosting/HeadlessProcessScheduler.cs
View file

@ -4,6 +4,9 @@ namespace AcDream.Headless.Hosting;
internal sealed class HeadlessProcessScheduler
{
private static readonly TimeSpan MinimumTimerDelay =
TimeSpan.FromMilliseconds(1);
private sealed class Slot
{
internal Slot(
@ -25,15 +28,23 @@ internal sealed class HeadlessProcessScheduler
private readonly long _periodTicks;
private readonly int _maximumCatchUpTurns;
private readonly Slot[] _slots;
private readonly Action<HeadlessSchedulerSnapshot>? _observation;
private readonly long _observationPeriodTicks;
private long _nextObservationDeadline;
private long _waitCount;
private long _turnCount;
private long _catchUpCollapseCount;
private long _lateDeadlineCount;
private long _totalLatenessTicks;
private long _maximumLatenessTicks;
internal HeadlessProcessScheduler(
IReadOnlyList<HeadlessSessionHost> sessions,
TimeProvider? timeProvider = null,
TimeSpan? turnPeriod = null,
int maximumCatchUpTurns = 8)
int maximumCatchUpTurns = 8,
Action<HeadlessSchedulerSnapshot>? observation = null,
TimeSpan? observationPeriod = null)
{
ArgumentNullException.ThrowIfNull(sessions);
if (sessions.Count == 0)
@ -57,8 +68,22 @@ internal sealed class HeadlessProcessScheduler
_timeProvider,
configuredTurnPeriod);
_maximumCatchUpTurns = maximumCatchUpTurns;
_observation = observation;
TimeSpan configuredObservationPeriod =
observationPeriod ?? TimeSpan.FromSeconds(30);
if (configuredObservationPeriod <= TimeSpan.Zero)
{
throw new ArgumentOutOfRangeException(
nameof(observationPeriod));
}
_observationPeriodTicks = DurationToTimestampTicks(
_timeProvider,
configuredObservationPeriod);
long start = _timeProvider.GetTimestamp();
_nextObservationDeadline = AddTicks(
start,
_observationPeriodTicks);
_slots = new Slot[sessions.Count];
for (int index = 0; index < sessions.Count; index++)
{
@ -89,24 +114,60 @@ internal sealed class HeadlessProcessScheduler
Interlocked.Read(ref _waitCount),
Interlocked.Read(ref _turnCount),
Interlocked.Read(ref _catchUpCollapseCount),
Interlocked.Read(ref _lateDeadlineCount),
Interlocked.Read(ref _totalLatenessTicks),
Interlocked.Read(ref _maximumLatenessTicks),
_timeProvider.TimestampFrequency,
activeSessionCount,
faultedSessionCount,
NextDeadline());
}
internal void RebaseDeadlinesAfterSessionStart()
{
if (Interlocked.Read(ref _waitCount) != 0L
|| Interlocked.Read(ref _turnCount) != 0L
|| Interlocked.Read(ref _catchUpCollapseCount) != 0L
|| Interlocked.Read(ref _lateDeadlineCount) != 0L)
{
throw new InvalidOperationException(
"A running headless scheduler cannot rebase its deadlines.");
}
long start = _timeProvider.GetTimestamp();
for (int index = 0; index < _slots.Length; index++)
{
_slots[index].LastTurnTimestamp = start;
_slots[index].NextDeadline = AddTicks(
start,
_periodTicks);
}
_nextObservationDeadline = AddTicks(
start,
_observationPeriodTicks);
}
internal async Task RunAsync(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested
&& HasActiveSession())
{
long now = _timeProvider.GetTimestamp();
if (DispatchDue(now))
bool dispatched = DispatchDue(now);
bool observed = DispatchObservationDue(now);
if (dispatched || observed)
continue;
long deadline = NextDeadline();
if (_observation is not null
&& _nextObservationDeadline < deadline)
{
deadline = _nextObservationDeadline;
}
TimeSpan delay = deadline <= now
? TimeSpan.Zero
: _timeProvider.GetElapsedTime(now, deadline);
delay = NormalizeTimerDelay(delay);
Interlocked.Increment(ref _waitCount);
await Task.Delay(
delay,
@ -116,6 +177,12 @@ internal sealed class HeadlessProcessScheduler
}
}
internal static TimeSpan NormalizeTimerDelay(TimeSpan delay) =>
delay > TimeSpan.Zero
&& delay < MinimumTimerDelay
? MinimumTimerDelay
: delay;
internal bool DispatchDue(long nowTimestamp)
{
bool dispatched = false;
@ -132,6 +199,9 @@ internal sealed class HeadlessProcessScheduler
if (nowTimestamp < session.ReconnectDeadline)
continue;
RecordLateness(
nowTimestamp,
session.ReconnectDeadline);
RuntimeSessionStartResult result =
session.CompletePendingReconnect(nowTimestamp);
if (result.Status
@ -164,8 +234,32 @@ internal sealed class HeadlessProcessScheduler
return dispatched;
}
internal bool DispatchObservationDue(long nowTimestamp)
{
if (_observation is null
|| nowTimestamp < _nextObservationDeadline)
{
return false;
}
long dueCount =
1L
+ ((nowTimestamp - _nextObservationDeadline)
/ _observationPeriodTicks);
long advance = dueCount
> long.MaxValue / _observationPeriodTicks
? long.MaxValue
: dueCount * _observationPeriodTicks;
_nextObservationDeadline = AddTicks(
_nextObservationDeadline,
advance);
_observation(CaptureSnapshot());
return true;
}
private void DispatchSessionDue(Slot slot, long nowTimestamp)
{
RecordLateness(nowTimestamp, slot.NextDeadline);
long dueCount =
1L + ((nowTimestamp - slot.NextDeadline) / _periodTicks);
int turns = (int)Math.Min(
@ -203,6 +297,27 @@ internal sealed class HeadlessProcessScheduler
Interlocked.Increment(ref _catchUpCollapseCount);
}
private void RecordLateness(long nowTimestamp, long deadline)
{
if (nowTimestamp <= deadline)
return;
long lateness = nowTimestamp - deadline;
Interlocked.Increment(ref _lateDeadlineCount);
Interlocked.Add(ref _totalLatenessTicks, lateness);
long maximum = Interlocked.Read(ref _maximumLatenessTicks);
while (lateness > maximum)
{
long observed = Interlocked.CompareExchange(
ref _maximumLatenessTicks,
lateness,
maximum);
if (observed == maximum)
break;
maximum = observed;
}
}
private long NextDeadline()
{
long next = long.MaxValue;
@ -257,6 +372,25 @@ internal readonly record struct HeadlessSchedulerSnapshot(
long WaitCount,
long TurnCount,
long CatchUpCollapseCount,
long LateDeadlineCount,
long TotalLatenessTicks,
long MaximumLatenessTicks,
long TimestampFrequency,
int ActiveSessionCount,
int FaultedSessionCount,
long NextDeadline);
long NextDeadline)
{
internal double MeanLatenessMilliseconds =>
LateDeadlineCount == 0
? 0d
: TimestampTicksToMilliseconds(
TotalLatenessTicks / (double)LateDeadlineCount);
internal double MaximumLatenessMilliseconds =>
TimestampTicksToMilliseconds(MaximumLatenessTicks);
private double TimestampTicksToMilliseconds(double ticks) =>
TimestampFrequency <= 0L
? 0d
: ticks * 1000d / TimestampFrequency;
}