perf(headless): enforce K4 resource envelope
This commit is contained in:
Erik
parent
bd236ce553
commit
776482da82
6 changed files with 797 additions and 2 deletions
|
|
@ -79,6 +79,8 @@ internal sealed class HeadlessDiagnosticWriter
|
||||||
{
|
{
|
||||||
ArgumentException.ThrowIfNullOrWhiteSpace(state);
|
ArgumentException.ThrowIfNullOrWhiteSpace(state);
|
||||||
HeadlessProcessContentSnapshot? content = snapshot.Content;
|
HeadlessProcessContentSnapshot? content = snapshot.Content;
|
||||||
|
HeadlessProcessResourceEnvelopeSnapshot envelope =
|
||||||
|
snapshot.Envelope;
|
||||||
Write(new
|
Write(new
|
||||||
{
|
{
|
||||||
kind = "resources",
|
kind = "resources",
|
||||||
|
|
@ -111,6 +113,54 @@ internal sealed class HeadlessDiagnosticWriter
|
||||||
content.Value.MappedVirtualBytes,
|
content.Value.MappedVirtualBytes,
|
||||||
content.Value.IsConverged,
|
content.Value.IsConverged,
|
||||||
},
|
},
|
||||||
|
resourceEnvelope = new
|
||||||
|
{
|
||||||
|
envelope.Ceilings.Profile,
|
||||||
|
envelope.HasRateSample,
|
||||||
|
envelope.WithinCeilings,
|
||||||
|
formula = new
|
||||||
|
{
|
||||||
|
envelope.Ceilings.SharedPrivateBytes,
|
||||||
|
envelope.Ceilings.PrivateBytesPerSession,
|
||||||
|
envelope.Ceilings.SharedWorkingSetBytes,
|
||||||
|
envelope.Ceilings.WorkingSetBytesPerSession,
|
||||||
|
envelope.Ceilings.SharedManagedLiveBytes,
|
||||||
|
envelope.Ceilings.ManagedLiveBytesPerSession,
|
||||||
|
envelope.Ceilings.SharedManagedHeapBytes,
|
||||||
|
envelope.Ceilings.ManagedHeapBytesPerSession,
|
||||||
|
envelope.Ceilings.SharedCpuCorePercent,
|
||||||
|
envelope.Ceilings.CpuCorePercentPerSession,
|
||||||
|
envelope.Ceilings.SharedHandleCount,
|
||||||
|
envelope.Ceilings.HandleCountPerSession,
|
||||||
|
envelope.Ceilings.SharedDescriptorCount,
|
||||||
|
envelope.Ceilings.DescriptorCountPerSession,
|
||||||
|
envelope.Ceilings.SharedSocketCount,
|
||||||
|
envelope.Ceilings.SocketCountPerSession,
|
||||||
|
},
|
||||||
|
limits = new
|
||||||
|
{
|
||||||
|
envelope.MaximumPrivateBytes,
|
||||||
|
envelope.MaximumWorkingSetBytes,
|
||||||
|
envelope.MaximumManagedLiveBytes,
|
||||||
|
envelope.MaximumManagedHeapBytes,
|
||||||
|
envelope.MaximumCpuCorePercent,
|
||||||
|
envelope.MaximumThreadCount,
|
||||||
|
envelope.MaximumHandleCount,
|
||||||
|
envelope.MaximumDescriptorCount,
|
||||||
|
envelope.MaximumSocketCount,
|
||||||
|
envelope.MaximumWaitsPerSecond,
|
||||||
|
envelope.MaximumCatchUpsPerSecond,
|
||||||
|
envelope.MaximumMeanLatenessMilliseconds,
|
||||||
|
envelope.MaximumLatenessMilliseconds,
|
||||||
|
},
|
||||||
|
observedRates = new
|
||||||
|
{
|
||||||
|
envelope.WaitsPerSecond,
|
||||||
|
envelope.TurnsPerSecond,
|
||||||
|
envelope.CatchUpsPerSecond,
|
||||||
|
},
|
||||||
|
envelope.Violations,
|
||||||
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,396 @@
|
||||||
|
using AcDream.Headless.Hosting;
|
||||||
|
|
||||||
|
namespace AcDream.Headless.Diagnostics;
|
||||||
|
|
||||||
|
internal readonly record struct HeadlessProcessResourceCeilings(
|
||||||
|
string Profile,
|
||||||
|
long SharedPrivateBytes,
|
||||||
|
long PrivateBytesPerSession,
|
||||||
|
long SharedWorkingSetBytes,
|
||||||
|
long WorkingSetBytesPerSession,
|
||||||
|
long SharedManagedLiveBytes,
|
||||||
|
long ManagedLiveBytesPerSession,
|
||||||
|
long SharedManagedHeapBytes,
|
||||||
|
long ManagedHeapBytesPerSession,
|
||||||
|
double SharedCpuCorePercent,
|
||||||
|
double CpuCorePercentPerSession,
|
||||||
|
int MaximumThreadCount,
|
||||||
|
int SharedHandleCount,
|
||||||
|
int HandleCountPerSession,
|
||||||
|
int SharedDescriptorCount,
|
||||||
|
int DescriptorCountPerSession,
|
||||||
|
int SharedSocketCount,
|
||||||
|
int SocketCountPerSession,
|
||||||
|
double MaximumWaitsPerSecond,
|
||||||
|
double MaximumCatchUpsPerSecond,
|
||||||
|
double MaximumMeanLatenessMilliseconds,
|
||||||
|
double MaximumLatenessMilliseconds)
|
||||||
|
{
|
||||||
|
internal const long MiB = 1024L * 1024L;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Conservative K4 Linux envelope derived from the fixed-binary native
|
||||||
|
/// two-session Rynthid baseline and the automated 1/5/10/30-root gates.
|
||||||
|
/// It is diagnostic-only: violations are observable but never alter
|
||||||
|
/// simulation, network, or teardown behavior.
|
||||||
|
/// </summary>
|
||||||
|
internal static HeadlessProcessResourceCeilings K4Linux { get; } =
|
||||||
|
new(
|
||||||
|
Profile: "k4-linux-30-session",
|
||||||
|
SharedPrivateBytes: 224L * MiB,
|
||||||
|
PrivateBytesPerSession: 64L * MiB,
|
||||||
|
SharedWorkingSetBytes: 224L * MiB,
|
||||||
|
WorkingSetBytesPerSession: 64L * MiB,
|
||||||
|
SharedManagedLiveBytes: 64L * MiB,
|
||||||
|
ManagedLiveBytesPerSession: 48L * MiB,
|
||||||
|
SharedManagedHeapBytes: 64L * MiB,
|
||||||
|
ManagedHeapBytesPerSession: 48L * MiB,
|
||||||
|
SharedCpuCorePercent: 2d,
|
||||||
|
CpuCorePercentPerSession: 3d,
|
||||||
|
MaximumThreadCount: 48,
|
||||||
|
SharedHandleCount: 128,
|
||||||
|
HandleCountPerSession: 3,
|
||||||
|
SharedDescriptorCount: 128,
|
||||||
|
DescriptorCountPerSession: 3,
|
||||||
|
SharedSocketCount: 1,
|
||||||
|
SocketCountPerSession: 2,
|
||||||
|
MaximumWaitsPerSecond: 250d,
|
||||||
|
MaximumCatchUpsPerSecond: 1d,
|
||||||
|
MaximumMeanLatenessMilliseconds: 10d,
|
||||||
|
MaximumLatenessMilliseconds: 750d);
|
||||||
|
|
||||||
|
internal long MaximumPrivateBytes(int sessionCount) =>
|
||||||
|
Scale(SharedPrivateBytes, PrivateBytesPerSession, sessionCount);
|
||||||
|
|
||||||
|
internal long MaximumWorkingSetBytes(int sessionCount) =>
|
||||||
|
Scale(SharedWorkingSetBytes, WorkingSetBytesPerSession, sessionCount);
|
||||||
|
|
||||||
|
internal long MaximumManagedLiveBytes(int sessionCount) =>
|
||||||
|
Scale(
|
||||||
|
SharedManagedLiveBytes,
|
||||||
|
ManagedLiveBytesPerSession,
|
||||||
|
sessionCount);
|
||||||
|
|
||||||
|
internal long MaximumManagedHeapBytes(int sessionCount) =>
|
||||||
|
Scale(
|
||||||
|
SharedManagedHeapBytes,
|
||||||
|
ManagedHeapBytesPerSession,
|
||||||
|
sessionCount);
|
||||||
|
|
||||||
|
internal double MaximumCpuCorePercent(int sessionCount) =>
|
||||||
|
SharedCpuCorePercent
|
||||||
|
+ CpuCorePercentPerSession * Math.Max(0, sessionCount);
|
||||||
|
|
||||||
|
internal int MaximumHandleCount(int sessionCount) =>
|
||||||
|
Scale(SharedHandleCount, HandleCountPerSession, sessionCount);
|
||||||
|
|
||||||
|
internal int MaximumDescriptorCount(int sessionCount) =>
|
||||||
|
Scale(
|
||||||
|
SharedDescriptorCount,
|
||||||
|
DescriptorCountPerSession,
|
||||||
|
sessionCount);
|
||||||
|
|
||||||
|
internal int MaximumSocketCount(int sessionCount) =>
|
||||||
|
Scale(SharedSocketCount, SocketCountPerSession, sessionCount);
|
||||||
|
|
||||||
|
private static long Scale(
|
||||||
|
long shared,
|
||||||
|
long perSession,
|
||||||
|
int sessionCount) =>
|
||||||
|
checked(shared + perSession * Math.Max(0, sessionCount));
|
||||||
|
|
||||||
|
private static int Scale(
|
||||||
|
int shared,
|
||||||
|
int perSession,
|
||||||
|
int sessionCount) =>
|
||||||
|
checked(shared + perSession * Math.Max(0, sessionCount));
|
||||||
|
}
|
||||||
|
|
||||||
|
internal readonly record struct HeadlessProcessResourceEnvelopeSnapshot(
|
||||||
|
HeadlessProcessResourceCeilings Ceilings,
|
||||||
|
bool HasRateSample,
|
||||||
|
bool WithinCeilings,
|
||||||
|
long MaximumPrivateBytes,
|
||||||
|
long MaximumWorkingSetBytes,
|
||||||
|
long MaximumManagedLiveBytes,
|
||||||
|
long MaximumManagedHeapBytes,
|
||||||
|
double MaximumCpuCorePercent,
|
||||||
|
int MaximumThreadCount,
|
||||||
|
int MaximumHandleCount,
|
||||||
|
int MaximumDescriptorCount,
|
||||||
|
int MaximumSocketCount,
|
||||||
|
double MaximumWaitsPerSecond,
|
||||||
|
double MaximumCatchUpsPerSecond,
|
||||||
|
double MaximumMeanLatenessMilliseconds,
|
||||||
|
double MaximumLatenessMilliseconds,
|
||||||
|
double WaitsPerSecond,
|
||||||
|
double TurnsPerSecond,
|
||||||
|
double CatchUpsPerSecond,
|
||||||
|
string[] Violations);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Stateful low-frequency evaluator for the committed K4 resource envelope.
|
||||||
|
/// It owns only prior cumulative counters needed to calculate interval rates.
|
||||||
|
/// </summary>
|
||||||
|
internal sealed class HeadlessProcessResourceEnvelopeEvaluator
|
||||||
|
{
|
||||||
|
private readonly HeadlessProcessResourceCeilings _ceilings;
|
||||||
|
private long _previousWaitCount;
|
||||||
|
private long _previousTurnCount;
|
||||||
|
private long _previousCatchUpCount;
|
||||||
|
private bool _hasPreviousCounters;
|
||||||
|
|
||||||
|
internal HeadlessProcessResourceEnvelopeEvaluator(
|
||||||
|
HeadlessProcessResourceCeilings? ceilings = null)
|
||||||
|
{
|
||||||
|
_ceilings = ceilings
|
||||||
|
?? HeadlessProcessResourceCeilings.K4Linux;
|
||||||
|
}
|
||||||
|
|
||||||
|
internal HeadlessProcessResourceEnvelopeSnapshot Evaluate(
|
||||||
|
string state,
|
||||||
|
in HeadlessProcessUsageSnapshot process,
|
||||||
|
in HeadlessManagedUsageSnapshot managed,
|
||||||
|
in HeadlessSessionUsageSnapshot sessions,
|
||||||
|
in HeadlessSchedulerSnapshot scheduler,
|
||||||
|
HeadlessProcessContentSnapshot? content)
|
||||||
|
{
|
||||||
|
ArgumentException.ThrowIfNullOrWhiteSpace(state);
|
||||||
|
int sessionCount = sessions.ConfiguredCount;
|
||||||
|
long maximumPrivateBytes =
|
||||||
|
_ceilings.MaximumPrivateBytes(sessionCount);
|
||||||
|
long maximumWorkingSetBytes =
|
||||||
|
_ceilings.MaximumWorkingSetBytes(sessionCount);
|
||||||
|
long maximumManagedLiveBytes =
|
||||||
|
_ceilings.MaximumManagedLiveBytes(sessionCount);
|
||||||
|
long maximumManagedHeapBytes =
|
||||||
|
_ceilings.MaximumManagedHeapBytes(sessionCount);
|
||||||
|
double maximumCpuCorePercent =
|
||||||
|
_ceilings.MaximumCpuCorePercent(sessionCount);
|
||||||
|
int maximumHandleCount =
|
||||||
|
_ceilings.MaximumHandleCount(sessionCount);
|
||||||
|
int maximumDescriptorCount =
|
||||||
|
_ceilings.MaximumDescriptorCount(sessionCount);
|
||||||
|
int maximumSocketCount =
|
||||||
|
_ceilings.MaximumSocketCount(sessionCount);
|
||||||
|
|
||||||
|
bool hasRateSample =
|
||||||
|
_hasPreviousCounters
|
||||||
|
&& process.SampleIntervalMilliseconds > 0d;
|
||||||
|
double intervalSeconds =
|
||||||
|
process.SampleIntervalMilliseconds / 1000d;
|
||||||
|
double waitsPerSecond = hasRateSample
|
||||||
|
? Delta(scheduler.WaitCount, _previousWaitCount)
|
||||||
|
/ intervalSeconds
|
||||||
|
: 0d;
|
||||||
|
double turnsPerSecond = hasRateSample
|
||||||
|
? Delta(scheduler.TurnCount, _previousTurnCount)
|
||||||
|
/ intervalSeconds
|
||||||
|
: 0d;
|
||||||
|
double catchUpsPerSecond = hasRateSample
|
||||||
|
? Delta(
|
||||||
|
scheduler.CatchUpCollapseCount,
|
||||||
|
_previousCatchUpCount)
|
||||||
|
/ intervalSeconds
|
||||||
|
: 0d;
|
||||||
|
_previousWaitCount = scheduler.WaitCount;
|
||||||
|
_previousTurnCount = scheduler.TurnCount;
|
||||||
|
_previousCatchUpCount = scheduler.CatchUpCollapseCount;
|
||||||
|
_hasPreviousCounters = true;
|
||||||
|
|
||||||
|
var violations = new List<string>();
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"private-bytes",
|
||||||
|
process.PrivateBytes,
|
||||||
|
maximumPrivateBytes);
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"working-set-bytes",
|
||||||
|
process.WorkingSetBytes,
|
||||||
|
maximumWorkingSetBytes);
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"managed-live-bytes",
|
||||||
|
managed.LiveBytes,
|
||||||
|
maximumManagedLiveBytes);
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"managed-heap-bytes",
|
||||||
|
managed.HeapSizeBytes,
|
||||||
|
maximumManagedHeapBytes);
|
||||||
|
if (process.SampleIntervalMilliseconds > 0d)
|
||||||
|
{
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"cpu-core-percent",
|
||||||
|
process.CpuCorePercent,
|
||||||
|
maximumCpuCorePercent);
|
||||||
|
}
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"thread-count",
|
||||||
|
process.ThreadCount,
|
||||||
|
_ceilings.MaximumThreadCount);
|
||||||
|
AddIfSupportedAbove(
|
||||||
|
violations,
|
||||||
|
"handle-count",
|
||||||
|
process.HandleCount,
|
||||||
|
maximumHandleCount);
|
||||||
|
AddIfSupportedAbove(
|
||||||
|
violations,
|
||||||
|
"descriptor-count",
|
||||||
|
process.FileDescriptorCount,
|
||||||
|
maximumDescriptorCount);
|
||||||
|
AddIfSupportedAbove(
|
||||||
|
violations,
|
||||||
|
"socket-count",
|
||||||
|
process.SocketDescriptorCount,
|
||||||
|
maximumSocketCount);
|
||||||
|
if (hasRateSample)
|
||||||
|
{
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"scheduler-waits-per-second",
|
||||||
|
waitsPerSecond,
|
||||||
|
_ceilings.MaximumWaitsPerSecond);
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"scheduler-catchups-per-second",
|
||||||
|
catchUpsPerSecond,
|
||||||
|
_ceilings.MaximumCatchUpsPerSecond);
|
||||||
|
}
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"scheduler-mean-lateness-milliseconds",
|
||||||
|
scheduler.MeanLatenessMilliseconds,
|
||||||
|
_ceilings.MaximumMeanLatenessMilliseconds);
|
||||||
|
AddIfAbove(
|
||||||
|
violations,
|
||||||
|
"scheduler-maximum-lateness-milliseconds",
|
||||||
|
scheduler.MaximumLatenessMilliseconds,
|
||||||
|
_ceilings.MaximumLatenessMilliseconds);
|
||||||
|
if (sessions.FaultedCount != 0
|
||||||
|
|| scheduler.FaultedSessionCount != 0)
|
||||||
|
{
|
||||||
|
violations.Add("faulted-session-debt");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (string.Equals(
|
||||||
|
state,
|
||||||
|
"disposed",
|
||||||
|
StringComparison.Ordinal))
|
||||||
|
{
|
||||||
|
EvaluateTerminal(
|
||||||
|
violations,
|
||||||
|
sessions,
|
||||||
|
content);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
EvaluateRunning(
|
||||||
|
violations,
|
||||||
|
sessions,
|
||||||
|
scheduler,
|
||||||
|
content);
|
||||||
|
}
|
||||||
|
|
||||||
|
return new HeadlessProcessResourceEnvelopeSnapshot(
|
||||||
|
_ceilings,
|
||||||
|
hasRateSample,
|
||||||
|
violations.Count == 0,
|
||||||
|
maximumPrivateBytes,
|
||||||
|
maximumWorkingSetBytes,
|
||||||
|
maximumManagedLiveBytes,
|
||||||
|
maximumManagedHeapBytes,
|
||||||
|
maximumCpuCorePercent,
|
||||||
|
_ceilings.MaximumThreadCount,
|
||||||
|
maximumHandleCount,
|
||||||
|
maximumDescriptorCount,
|
||||||
|
maximumSocketCount,
|
||||||
|
_ceilings.MaximumWaitsPerSecond,
|
||||||
|
_ceilings.MaximumCatchUpsPerSecond,
|
||||||
|
_ceilings.MaximumMeanLatenessMilliseconds,
|
||||||
|
_ceilings.MaximumLatenessMilliseconds,
|
||||||
|
waitsPerSecond,
|
||||||
|
turnsPerSecond,
|
||||||
|
catchUpsPerSecond,
|
||||||
|
[.. violations]);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void EvaluateRunning(
|
||||||
|
List<string> violations,
|
||||||
|
in HeadlessSessionUsageSnapshot sessions,
|
||||||
|
in HeadlessSchedulerSnapshot scheduler,
|
||||||
|
HeadlessProcessContentSnapshot? content)
|
||||||
|
{
|
||||||
|
if (scheduler.SessionCount != sessions.ConfiguredCount)
|
||||||
|
violations.Add("scheduler-session-count");
|
||||||
|
if (sessions.HostLeaseCount != sessions.ConfiguredCount)
|
||||||
|
violations.Add("runtime-host-lease-debt");
|
||||||
|
if (sessions.InWorldCount + sessions.PendingReconnectCount
|
||||||
|
< sessions.ConfiguredCount)
|
||||||
|
{
|
||||||
|
violations.Add("unaccounted-session-lifecycle");
|
||||||
|
}
|
||||||
|
if (content is { } current
|
||||||
|
&& (current.LeaseCount != sessions.ConfiguredCount
|
||||||
|
|| current.IsDisposeRequested
|
||||||
|
|| current.IsDisposed))
|
||||||
|
{
|
||||||
|
violations.Add("shared-content-lease-debt");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void EvaluateTerminal(
|
||||||
|
List<string> violations,
|
||||||
|
in HeadlessSessionUsageSnapshot sessions,
|
||||||
|
HeadlessProcessContentSnapshot? content)
|
||||||
|
{
|
||||||
|
if (sessions.InWorldCount != 0
|
||||||
|
|| sessions.PendingReconnectCount != 0
|
||||||
|
|| sessions.ConvergedRuntimeCount
|
||||||
|
!= sessions.ConfiguredCount
|
||||||
|
|| sessions.EntityCount != 0
|
||||||
|
|| sessions.InventoryObjectCount != 0
|
||||||
|
|| sessions.HostLeaseCount != 0)
|
||||||
|
{
|
||||||
|
violations.Add("runtime-terminal-debt");
|
||||||
|
}
|
||||||
|
if (content is { IsConverged: false })
|
||||||
|
violations.Add("shared-content-terminal-debt");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static long Delta(long current, long previous) =>
|
||||||
|
Math.Max(0L, current - previous);
|
||||||
|
|
||||||
|
private static void AddIfSupportedAbove(
|
||||||
|
List<string> violations,
|
||||||
|
string name,
|
||||||
|
int value,
|
||||||
|
int ceiling)
|
||||||
|
{
|
||||||
|
if (value >= 0)
|
||||||
|
AddIfAbove(violations, name, value, ceiling);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AddIfAbove(
|
||||||
|
List<string> violations,
|
||||||
|
string name,
|
||||||
|
long value,
|
||||||
|
long ceiling)
|
||||||
|
{
|
||||||
|
if (value > ceiling)
|
||||||
|
violations.Add(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void AddIfAbove(
|
||||||
|
List<string> violations,
|
||||||
|
string name,
|
||||||
|
double value,
|
||||||
|
double ceiling)
|
||||||
|
{
|
||||||
|
if (!double.IsFinite(value) || value > ceiling)
|
||||||
|
violations.Add(name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -48,7 +48,8 @@ internal readonly record struct HeadlessProcessResourceSnapshot(
|
||||||
HeadlessManagedUsageSnapshot Managed,
|
HeadlessManagedUsageSnapshot Managed,
|
||||||
HeadlessSessionUsageSnapshot Sessions,
|
HeadlessSessionUsageSnapshot Sessions,
|
||||||
HeadlessSchedulerSnapshot Scheduler,
|
HeadlessSchedulerSnapshot Scheduler,
|
||||||
HeadlessProcessContentSnapshot? Content);
|
HeadlessProcessContentSnapshot? Content,
|
||||||
|
HeadlessProcessResourceEnvelopeSnapshot Envelope);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Captures one low-frequency process-wide resource sample. The sampler owns
|
/// Captures one low-frequency process-wide resource sample. The sampler owns
|
||||||
|
|
@ -58,6 +59,7 @@ internal sealed class HeadlessProcessResourceSampler : IDisposable
|
||||||
{
|
{
|
||||||
private readonly TimeProvider _timeProvider;
|
private readonly TimeProvider _timeProvider;
|
||||||
private readonly Process _process;
|
private readonly Process _process;
|
||||||
|
private readonly HeadlessProcessResourceEnvelopeEvaluator _envelope;
|
||||||
private long _sequence;
|
private long _sequence;
|
||||||
private long _previousTimestamp;
|
private long _previousTimestamp;
|
||||||
private long _previousProcessorTicks;
|
private long _previousProcessorTicks;
|
||||||
|
|
@ -70,14 +72,17 @@ internal sealed class HeadlessProcessResourceSampler : IDisposable
|
||||||
{
|
{
|
||||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||||
_process = process ?? Process.GetCurrentProcess();
|
_process = process ?? Process.GetCurrentProcess();
|
||||||
|
_envelope = new HeadlessProcessResourceEnvelopeEvaluator();
|
||||||
}
|
}
|
||||||
|
|
||||||
internal HeadlessProcessResourceSnapshot Capture(
|
internal HeadlessProcessResourceSnapshot Capture(
|
||||||
|
string state,
|
||||||
IReadOnlyList<HeadlessSessionHost> sessions,
|
IReadOnlyList<HeadlessSessionHost> sessions,
|
||||||
HeadlessSchedulerSnapshot scheduler,
|
HeadlessSchedulerSnapshot scheduler,
|
||||||
HeadlessProcessContentSnapshot? content)
|
HeadlessProcessContentSnapshot? content)
|
||||||
{
|
{
|
||||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||||
|
ArgumentException.ThrowIfNullOrWhiteSpace(state);
|
||||||
ArgumentNullException.ThrowIfNull(sessions);
|
ArgumentNullException.ThrowIfNull(sessions);
|
||||||
|
|
||||||
long now = _timeProvider.GetTimestamp();
|
long now = _timeProvider.GetTimestamp();
|
||||||
|
|
@ -179,6 +184,14 @@ internal sealed class HeadlessProcessResourceSampler : IDisposable
|
||||||
entityCount,
|
entityCount,
|
||||||
inventoryObjectCount,
|
inventoryObjectCount,
|
||||||
hostLeaseCount);
|
hostLeaseCount);
|
||||||
|
HeadlessProcessResourceEnvelopeSnapshot envelope =
|
||||||
|
_envelope.Evaluate(
|
||||||
|
state,
|
||||||
|
process,
|
||||||
|
managed,
|
||||||
|
sessionUsage,
|
||||||
|
scheduler,
|
||||||
|
content);
|
||||||
return new HeadlessProcessResourceSnapshot(
|
return new HeadlessProcessResourceSnapshot(
|
||||||
checked(++_sequence),
|
checked(++_sequence),
|
||||||
now,
|
now,
|
||||||
|
|
@ -186,7 +199,8 @@ internal sealed class HeadlessProcessResourceSampler : IDisposable
|
||||||
managed,
|
managed,
|
||||||
sessionUsage,
|
sessionUsage,
|
||||||
scheduler,
|
scheduler,
|
||||||
content);
|
content,
|
||||||
|
envelope);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void Dispose()
|
public void Dispose()
|
||||||
|
|
|
||||||
|
|
@ -233,6 +233,7 @@ internal sealed class HeadlessProcessHost : IDisposable
|
||||||
HeadlessProcessContentOwner? content)
|
HeadlessProcessContentOwner? content)
|
||||||
{
|
{
|
||||||
HeadlessProcessResourceSnapshot snapshot = resources.Capture(
|
HeadlessProcessResourceSnapshot snapshot = resources.Capture(
|
||||||
|
state,
|
||||||
_sessions,
|
_sessions,
|
||||||
scheduler,
|
scheduler,
|
||||||
content?.CaptureSnapshot());
|
content?.CaptureSnapshot());
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,322 @@
|
||||||
|
using AcDream.Headless.Diagnostics;
|
||||||
|
using AcDream.Headless.Hosting;
|
||||||
|
|
||||||
|
namespace AcDream.Headless.Tests;
|
||||||
|
|
||||||
|
public sealed class HeadlessProcessResourceEnvelopeTests
|
||||||
|
{
|
||||||
|
[Fact]
|
||||||
|
public void K4ProfileScalesOnlyPerSessionDimensions()
|
||||||
|
{
|
||||||
|
HeadlessProcessResourceCeilings ceilings =
|
||||||
|
HeadlessProcessResourceCeilings.K4Linux;
|
||||||
|
|
||||||
|
Assert.Equal(
|
||||||
|
288L * HeadlessProcessResourceCeilings.MiB,
|
||||||
|
ceilings.MaximumPrivateBytes(1));
|
||||||
|
Assert.Equal(
|
||||||
|
352L * HeadlessProcessResourceCeilings.MiB,
|
||||||
|
ceilings.MaximumPrivateBytes(2));
|
||||||
|
Assert.Equal(
|
||||||
|
2144L * HeadlessProcessResourceCeilings.MiB,
|
||||||
|
ceilings.MaximumPrivateBytes(30));
|
||||||
|
Assert.Equal(8d, ceilings.MaximumCpuCorePercent(2));
|
||||||
|
Assert.Equal(92d, ceilings.MaximumCpuCorePercent(30));
|
||||||
|
Assert.Equal(
|
||||||
|
160L * HeadlessProcessResourceCeilings.MiB,
|
||||||
|
ceilings.MaximumManagedHeapBytes(2));
|
||||||
|
Assert.Equal(48, ceilings.MaximumThreadCount);
|
||||||
|
Assert.Equal(5, ceilings.MaximumSocketCount(2));
|
||||||
|
Assert.Equal(61, ceilings.MaximumSocketCount(30));
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void FixedBinaryTwoSessionRynthidSamplePasses()
|
||||||
|
{
|
||||||
|
var evaluator =
|
||||||
|
new HeadlessProcessResourceEnvelopeEvaluator();
|
||||||
|
_ = evaluator.Evaluate(
|
||||||
|
"running-start",
|
||||||
|
Process(
|
||||||
|
privateBytes: 207_966_208L,
|
||||||
|
workingSetBytes: 190_509_056L),
|
||||||
|
Managed(71_036_416L),
|
||||||
|
RunningSessions(2),
|
||||||
|
Scheduler(2),
|
||||||
|
RunningContent(2));
|
||||||
|
|
||||||
|
HeadlessProcessResourceEnvelopeSnapshot result =
|
||||||
|
evaluator.Evaluate(
|
||||||
|
"periodic",
|
||||||
|
Process(
|
||||||
|
privateBytes: 313_847_808L,
|
||||||
|
workingSetBytes: 302_817_280L,
|
||||||
|
cpuCorePercent: 6.2811d,
|
||||||
|
sampleIntervalMilliseconds: 30_000d,
|
||||||
|
threadCount: 17,
|
||||||
|
handleCount: 115,
|
||||||
|
descriptorCount: 116,
|
||||||
|
socketCount: 3),
|
||||||
|
Managed(122_179_536L),
|
||||||
|
RunningSessions(2),
|
||||||
|
Scheduler(
|
||||||
|
2,
|
||||||
|
waitCount: 2918L,
|
||||||
|
turnCount: 3974L,
|
||||||
|
catchUpCount: 2L,
|
||||||
|
lateCount: 3974L,
|
||||||
|
meanLatenessMilliseconds: 3.31d,
|
||||||
|
maximumLatenessMilliseconds: 302.16d),
|
||||||
|
RunningContent(2));
|
||||||
|
|
||||||
|
Assert.True(result.HasRateSample);
|
||||||
|
Assert.True(result.WithinCeilings);
|
||||||
|
Assert.Empty(result.Violations);
|
||||||
|
Assert.Equal(97.2667d, result.WaitsPerSecond, precision: 4);
|
||||||
|
Assert.Equal(132.4667d, result.TurnsPerSecond, precision: 4);
|
||||||
|
Assert.Equal(0.0667d, result.CatchUpsPerSecond, precision: 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void EveryResourceAndRuntimeDebtIsNamed()
|
||||||
|
{
|
||||||
|
var evaluator =
|
||||||
|
new HeadlessProcessResourceEnvelopeEvaluator();
|
||||||
|
_ = evaluator.Evaluate(
|
||||||
|
"running-start",
|
||||||
|
Process(1L, 1L),
|
||||||
|
Managed(1L),
|
||||||
|
RunningSessions(2),
|
||||||
|
Scheduler(2),
|
||||||
|
RunningContent(2));
|
||||||
|
HeadlessProcessResourceCeilings ceilings =
|
||||||
|
HeadlessProcessResourceCeilings.K4Linux;
|
||||||
|
var sessions = new HeadlessSessionUsageSnapshot(
|
||||||
|
ConfiguredCount: 2,
|
||||||
|
InWorldCount: 0,
|
||||||
|
FaultedCount: 1,
|
||||||
|
PendingReconnectCount: 0,
|
||||||
|
ConvergedRuntimeCount: 0,
|
||||||
|
EntityCount: 0,
|
||||||
|
InventoryObjectCount: 0,
|
||||||
|
HostLeaseCount: 0);
|
||||||
|
|
||||||
|
HeadlessProcessResourceEnvelopeSnapshot result =
|
||||||
|
evaluator.Evaluate(
|
||||||
|
"periodic",
|
||||||
|
Process(
|
||||||
|
ceilings.MaximumPrivateBytes(2) + 1L,
|
||||||
|
ceilings.MaximumWorkingSetBytes(2) + 1L,
|
||||||
|
cpuCorePercent:
|
||||||
|
ceilings.MaximumCpuCorePercent(2) + 0.01d,
|
||||||
|
sampleIntervalMilliseconds: 30_000d,
|
||||||
|
threadCount: ceilings.MaximumThreadCount + 1,
|
||||||
|
handleCount:
|
||||||
|
ceilings.MaximumHandleCount(2) + 1,
|
||||||
|
descriptorCount:
|
||||||
|
ceilings.MaximumDescriptorCount(2) + 1,
|
||||||
|
socketCount:
|
||||||
|
ceilings.MaximumSocketCount(2) + 1),
|
||||||
|
Managed(
|
||||||
|
ceilings.MaximumManagedLiveBytes(2) + 1L),
|
||||||
|
sessions,
|
||||||
|
Scheduler(
|
||||||
|
sessionCount: 1,
|
||||||
|
waitCount: 7501L,
|
||||||
|
turnCount: 1L,
|
||||||
|
catchUpCount: 31L,
|
||||||
|
lateCount: 1L,
|
||||||
|
meanLatenessMilliseconds: 10.01d,
|
||||||
|
maximumLatenessMilliseconds: 750.01d,
|
||||||
|
faultedCount: 1),
|
||||||
|
new HeadlessProcessContentSnapshot(
|
||||||
|
0,
|
||||||
|
IsDisposeRequested: true,
|
||||||
|
IsDisposed: false,
|
||||||
|
MappedVirtualBytes: 1L));
|
||||||
|
|
||||||
|
Assert.False(result.WithinCeilings);
|
||||||
|
string[] expected =
|
||||||
|
[
|
||||||
|
"private-bytes",
|
||||||
|
"working-set-bytes",
|
||||||
|
"managed-live-bytes",
|
||||||
|
"managed-heap-bytes",
|
||||||
|
"cpu-core-percent",
|
||||||
|
"thread-count",
|
||||||
|
"handle-count",
|
||||||
|
"descriptor-count",
|
||||||
|
"socket-count",
|
||||||
|
"scheduler-waits-per-second",
|
||||||
|
"scheduler-catchups-per-second",
|
||||||
|
"scheduler-mean-lateness-milliseconds",
|
||||||
|
"scheduler-maximum-lateness-milliseconds",
|
||||||
|
"faulted-session-debt",
|
||||||
|
"scheduler-session-count",
|
||||||
|
"runtime-host-lease-debt",
|
||||||
|
"unaccounted-session-lifecycle",
|
||||||
|
"shared-content-lease-debt",
|
||||||
|
];
|
||||||
|
Assert.Equal(expected, result.Violations);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void DisposedSampleRequiresExactRuntimeAndContentConvergence()
|
||||||
|
{
|
||||||
|
var failing =
|
||||||
|
new HeadlessProcessResourceEnvelopeEvaluator();
|
||||||
|
HeadlessProcessResourceEnvelopeSnapshot debt =
|
||||||
|
failing.Evaluate(
|
||||||
|
"disposed",
|
||||||
|
Process(1L, 1L),
|
||||||
|
Managed(1L),
|
||||||
|
RunningSessions(2),
|
||||||
|
Scheduler(2),
|
||||||
|
RunningContent(2));
|
||||||
|
|
||||||
|
Assert.False(debt.WithinCeilings);
|
||||||
|
Assert.Contains("runtime-terminal-debt", debt.Violations);
|
||||||
|
Assert.Contains(
|
||||||
|
"shared-content-terminal-debt",
|
||||||
|
debt.Violations);
|
||||||
|
|
||||||
|
var passing =
|
||||||
|
new HeadlessProcessResourceEnvelopeEvaluator();
|
||||||
|
HeadlessProcessResourceEnvelopeSnapshot converged =
|
||||||
|
passing.Evaluate(
|
||||||
|
"disposed",
|
||||||
|
Process(1L, 1L),
|
||||||
|
Managed(1L),
|
||||||
|
new HeadlessSessionUsageSnapshot(
|
||||||
|
ConfiguredCount: 2,
|
||||||
|
InWorldCount: 0,
|
||||||
|
FaultedCount: 0,
|
||||||
|
PendingReconnectCount: 0,
|
||||||
|
ConvergedRuntimeCount: 2,
|
||||||
|
EntityCount: 0,
|
||||||
|
InventoryObjectCount: 0,
|
||||||
|
HostLeaseCount: 0),
|
||||||
|
Scheduler(2),
|
||||||
|
new HeadlessProcessContentSnapshot(
|
||||||
|
0,
|
||||||
|
IsDisposeRequested: true,
|
||||||
|
IsDisposed: true,
|
||||||
|
MappedVirtualBytes: 0L));
|
||||||
|
|
||||||
|
Assert.True(converged.WithinCeilings);
|
||||||
|
Assert.Empty(converged.Violations);
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void NonFiniteResourceRatesFailClosed()
|
||||||
|
{
|
||||||
|
var evaluator =
|
||||||
|
new HeadlessProcessResourceEnvelopeEvaluator();
|
||||||
|
|
||||||
|
HeadlessProcessResourceEnvelopeSnapshot result =
|
||||||
|
evaluator.Evaluate(
|
||||||
|
"periodic",
|
||||||
|
Process(
|
||||||
|
1L,
|
||||||
|
1L,
|
||||||
|
cpuCorePercent: double.NaN,
|
||||||
|
sampleIntervalMilliseconds: 30_000d),
|
||||||
|
Managed(1L),
|
||||||
|
RunningSessions(1),
|
||||||
|
Scheduler(1),
|
||||||
|
RunningContent(1));
|
||||||
|
|
||||||
|
Assert.False(result.WithinCeilings);
|
||||||
|
Assert.Contains("cpu-core-percent", result.Violations);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static HeadlessProcessUsageSnapshot Process(
|
||||||
|
long privateBytes,
|
||||||
|
long workingSetBytes,
|
||||||
|
double cpuCorePercent = 0d,
|
||||||
|
double sampleIntervalMilliseconds = 0d,
|
||||||
|
int threadCount = 16,
|
||||||
|
int handleCount = 113,
|
||||||
|
int descriptorCount = 112,
|
||||||
|
int socketCount = 3) =>
|
||||||
|
new(
|
||||||
|
WorkingSetBytes: workingSetBytes,
|
||||||
|
PrivateBytes: privateBytes,
|
||||||
|
VirtualBytes: 30_000_000_000L,
|
||||||
|
TotalProcessorTimeTicks: 0L,
|
||||||
|
SampleIntervalMilliseconds: sampleIntervalMilliseconds,
|
||||||
|
CpuCorePercent: cpuCorePercent,
|
||||||
|
CpuMachinePercent: cpuCorePercent / 16d,
|
||||||
|
ProcessorCount: 16,
|
||||||
|
ThreadCount: threadCount,
|
||||||
|
HandleCount: handleCount,
|
||||||
|
FileDescriptorCount: descriptorCount,
|
||||||
|
SocketDescriptorCount: socketCount);
|
||||||
|
|
||||||
|
private static HeadlessManagedUsageSnapshot Managed(
|
||||||
|
long liveBytes) =>
|
||||||
|
new(
|
||||||
|
LiveBytes: liveBytes,
|
||||||
|
HeapSizeBytes: liveBytes,
|
||||||
|
FragmentedBytes: 0L,
|
||||||
|
CommittedBytes: liveBytes,
|
||||||
|
TotalAllocatedBytes: liveBytes,
|
||||||
|
MemoryLoadBytes: liveBytes,
|
||||||
|
HighMemoryLoadThresholdBytes: long.MaxValue,
|
||||||
|
PauseTimePercentage: 0d,
|
||||||
|
Gen0Collections: 0,
|
||||||
|
Gen1Collections: 0,
|
||||||
|
Gen2Collections: 0);
|
||||||
|
|
||||||
|
private static HeadlessSessionUsageSnapshot RunningSessions(
|
||||||
|
int sessionCount) =>
|
||||||
|
new(
|
||||||
|
ConfiguredCount: sessionCount,
|
||||||
|
InWorldCount: sessionCount,
|
||||||
|
FaultedCount: 0,
|
||||||
|
PendingReconnectCount: 0,
|
||||||
|
ConvergedRuntimeCount: 0,
|
||||||
|
EntityCount: sessionCount,
|
||||||
|
InventoryObjectCount: sessionCount,
|
||||||
|
HostLeaseCount: sessionCount);
|
||||||
|
|
||||||
|
private static HeadlessSchedulerSnapshot Scheduler(
|
||||||
|
int sessionCount,
|
||||||
|
long waitCount = 0L,
|
||||||
|
long turnCount = 0L,
|
||||||
|
long catchUpCount = 0L,
|
||||||
|
long lateCount = 0L,
|
||||||
|
double meanLatenessMilliseconds = 0d,
|
||||||
|
double maximumLatenessMilliseconds = 0d,
|
||||||
|
int faultedCount = 0)
|
||||||
|
{
|
||||||
|
const long frequency = TimeSpan.TicksPerSecond;
|
||||||
|
long totalLatenessTicks = checked((long)(
|
||||||
|
meanLatenessMilliseconds
|
||||||
|
* TimeSpan.TicksPerMillisecond
|
||||||
|
* lateCount));
|
||||||
|
long maximumLatenessTicks = checked((long)(
|
||||||
|
maximumLatenessMilliseconds
|
||||||
|
* TimeSpan.TicksPerMillisecond));
|
||||||
|
return new HeadlessSchedulerSnapshot(
|
||||||
|
SessionCount: sessionCount,
|
||||||
|
WaitCount: waitCount,
|
||||||
|
TurnCount: turnCount,
|
||||||
|
CatchUpCollapseCount: catchUpCount,
|
||||||
|
LateDeadlineCount: lateCount,
|
||||||
|
TotalLatenessTicks: totalLatenessTicks,
|
||||||
|
MaximumLatenessTicks: maximumLatenessTicks,
|
||||||
|
TimestampFrequency: frequency,
|
||||||
|
ActiveSessionCount: sessionCount - faultedCount,
|
||||||
|
FaultedSessionCount: faultedCount,
|
||||||
|
NextDeadline: 0L);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static HeadlessProcessContentSnapshot RunningContent(
|
||||||
|
int sessionCount) =>
|
||||||
|
new(
|
||||||
|
LeaseCount: sessionCount,
|
||||||
|
IsDisposeRequested: false,
|
||||||
|
IsDisposed: false,
|
||||||
|
MappedVirtualBytes: 29_908_271_024L);
|
||||||
|
}
|
||||||
|
|
@ -345,6 +345,14 @@ public sealed class HeadlessProcessSchedulerTests
|
||||||
"\"configuredCount\":2",
|
"\"configuredCount\":2",
|
||||||
diagnostics.ToString(),
|
diagnostics.ToString(),
|
||||||
StringComparison.Ordinal);
|
StringComparison.Ordinal);
|
||||||
|
Assert.Contains(
|
||||||
|
"\"resourceEnvelope\":",
|
||||||
|
diagnostics.ToString(),
|
||||||
|
StringComparison.Ordinal);
|
||||||
|
Assert.Contains(
|
||||||
|
"\"profile\":\"k4-linux-30-session\"",
|
||||||
|
diagnostics.ToString(),
|
||||||
|
StringComparison.Ordinal);
|
||||||
|
|
||||||
host.Dispose();
|
host.Dispose();
|
||||||
|
|
||||||
|
|
@ -356,6 +364,10 @@ public sealed class HeadlessProcessSchedulerTests
|
||||||
"\"convergedRuntimeCount\":2",
|
"\"convergedRuntimeCount\":2",
|
||||||
diagnostics.ToString(),
|
diagnostics.ToString(),
|
||||||
StringComparison.Ordinal);
|
StringComparison.Ordinal);
|
||||||
|
Assert.Contains(
|
||||||
|
"\"violations\":",
|
||||||
|
diagnostics.ToString(),
|
||||||
|
StringComparison.Ordinal);
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue