fix(launcher): guard orphan bake publication
This commit is contained in:
Erik
parent
3f68895120
commit
208a70ac83
14 changed files with 829 additions and 55 deletions
|
|
@ -8,4 +8,9 @@
|
|||
<IsPackable>false</IsPackable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\AcDream.Bake\AcDream.Bake.csproj" />
|
||||
<ProjectReference Include="..\..\src\AcDream.Launcher.Core\AcDream.Launcher.Core.csproj" />
|
||||
<ProjectReference Include="..\..\src\AcDream.Platform\AcDream.Platform.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -1,24 +1,190 @@
|
|||
if (args.Length != 3)
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using AcDream.Bake;
|
||||
using AcDream.Launcher.Core.Installation;
|
||||
using AcDream.Platform;
|
||||
|
||||
return args.FirstOrDefault() switch
|
||||
{
|
||||
return 2;
|
||||
"hold-install-lease" => await HoldInstallLeaseAsync(args[1..]),
|
||||
"orphan-parent" => await RunOrphanParentAsync(args[1..]),
|
||||
"orphan-child" => RunOrphanChild(args[1..]),
|
||||
_ => 2,
|
||||
};
|
||||
|
||||
static async Task<int> HoldInstallLeaseAsync(string[] arguments)
|
||||
{
|
||||
if (arguments.Length != 3)
|
||||
{
|
||||
return 2;
|
||||
}
|
||||
|
||||
string lockPath = Path.GetFullPath(arguments[0]);
|
||||
string stagingPath = Path.GetFullPath(arguments[1]);
|
||||
string readyPath = Path.GetFullPath(arguments[2]);
|
||||
Directory.CreateDirectory(
|
||||
Path.GetDirectoryName(lockPath)
|
||||
?? throw new InvalidOperationException("lock path has no parent"));
|
||||
Directory.CreateDirectory(
|
||||
Path.GetDirectoryName(stagingPath)
|
||||
?? throw new InvalidOperationException("staging path has no parent"));
|
||||
|
||||
using var lease = new FileStream(
|
||||
lockPath,
|
||||
FileMode.OpenOrCreate,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None);
|
||||
File.WriteAllText(stagingPath, "abandoned bake staging");
|
||||
File.WriteAllText(readyPath, "ready");
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan);
|
||||
return 0;
|
||||
}
|
||||
|
||||
string lockPath = Path.GetFullPath(args[0]);
|
||||
string stagingPath = Path.GetFullPath(args[1]);
|
||||
string readyPath = Path.GetFullPath(args[2]);
|
||||
Directory.CreateDirectory(
|
||||
Path.GetDirectoryName(lockPath)
|
||||
?? throw new InvalidOperationException("lock path has no parent"));
|
||||
Directory.CreateDirectory(
|
||||
Path.GetDirectoryName(stagingPath)
|
||||
?? throw new InvalidOperationException("staging path has no parent"));
|
||||
static async Task<int> RunOrphanParentAsync(string[] arguments)
|
||||
{
|
||||
if (arguments.Length != 8)
|
||||
{
|
||||
return 2;
|
||||
}
|
||||
|
||||
using var lease = new FileStream(
|
||||
lockPath,
|
||||
FileMode.OpenOrCreate,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None);
|
||||
File.WriteAllText(stagingPath, "abandoned bake staging");
|
||||
File.WriteAllText(readyPath, "ready");
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan);
|
||||
return 0;
|
||||
string dataDirectory = Path.GetFullPath(arguments[0]);
|
||||
string datDirectory = Path.GetFullPath(arguments[1]);
|
||||
string bakeMarker = Path.GetFullPath(arguments[2]);
|
||||
string schedule = arguments[3];
|
||||
string childReadyPath = Path.GetFullPath(arguments[4]);
|
||||
string childReleasePath = Path.GetFullPath(arguments[5]);
|
||||
string childPidPath = Path.GetFullPath(arguments[6]);
|
||||
string childExitPath = Path.GetFullPath(arguments[7]);
|
||||
var paths = new ApplicationPathSet(
|
||||
Path.Combine(dataDirectory, "fixture-config"),
|
||||
dataDirectory,
|
||||
Path.Combine(dataDirectory, "fixture-cache"),
|
||||
null);
|
||||
var runner = new OrphanBakeProcessRunner(
|
||||
schedule,
|
||||
childReadyPath,
|
||||
childReleasePath,
|
||||
childPidPath,
|
||||
childExitPath);
|
||||
var installer = new LauncherInstaller(
|
||||
paths,
|
||||
bakeMarker,
|
||||
processRunner: runner);
|
||||
|
||||
try
|
||||
{
|
||||
await installer.InstallAsync(datDirectory, 1);
|
||||
return 0;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return 9;
|
||||
}
|
||||
}
|
||||
|
||||
static int RunOrphanChild(string[] arguments)
|
||||
{
|
||||
if (arguments.Length != 5)
|
||||
{
|
||||
return 2;
|
||||
}
|
||||
|
||||
string outputPath = Path.GetFullPath(arguments[0]);
|
||||
string schedule = arguments[1];
|
||||
string readyPath = Path.GetFullPath(arguments[2]);
|
||||
string releasePath = Path.GetFullPath(arguments[3]);
|
||||
string exitPath = Path.GetFullPath(arguments[4]);
|
||||
Action barrier = () =>
|
||||
{
|
||||
File.WriteAllText(readyPath, schedule);
|
||||
while (!File.Exists(releasePath))
|
||||
{
|
||||
Thread.Sleep(10);
|
||||
}
|
||||
};
|
||||
|
||||
int exitCode;
|
||||
try
|
||||
{
|
||||
BakeOutputTransaction.WriteValidateAndPublish(
|
||||
outputPath,
|
||||
temporaryPath =>
|
||||
{
|
||||
File.WriteAllText(temporaryPath, "orphan replacement");
|
||||
return 1;
|
||||
},
|
||||
(temporaryPath, _) =>
|
||||
{
|
||||
if (File.ReadAllText(temporaryPath) != "orphan replacement")
|
||||
{
|
||||
throw new InvalidDataException("staging content changed");
|
||||
}
|
||||
},
|
||||
beforePublicationLock: schedule == "late" ? barrier : null,
|
||||
beforePromotion: schedule == "holds" ? barrier : null,
|
||||
CancellationToken.None);
|
||||
exitCode = 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
File.WriteAllText(exitPath + ".error", ex.Message);
|
||||
exitCode = 17;
|
||||
}
|
||||
|
||||
File.WriteAllText(exitPath, exitCode.ToString(
|
||||
System.Globalization.CultureInfo.InvariantCulture));
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
file sealed class OrphanBakeProcessRunner(
|
||||
string schedule,
|
||||
string childReadyPath,
|
||||
string childReleasePath,
|
||||
string childPidPath,
|
||||
string childExitPath) : IBakeProcessRunner
|
||||
{
|
||||
public async Task<BakeProcessResult> RunAsync(
|
||||
BakeProcessRequest request,
|
||||
Action<string> onStandardOutput,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string dotnetHost = Environment.ProcessPath
|
||||
?? throw new InvalidOperationException("dotnet host path is unavailable");
|
||||
string fixtureDll = Assembly.GetExecutingAssembly().Location;
|
||||
var startInfo = new ProcessStartInfo(dotnetHost)
|
||||
{
|
||||
UseShellExecute = false,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
startInfo.ArgumentList.Add(fixtureDll);
|
||||
startInfo.ArgumentList.Add("orphan-child");
|
||||
startInfo.ArgumentList.Add(request.OutputPath);
|
||||
startInfo.ArgumentList.Add(schedule);
|
||||
startInfo.ArgumentList.Add(childReadyPath);
|
||||
startInfo.ArgumentList.Add(childReleasePath);
|
||||
startInfo.ArgumentList.Add(childExitPath);
|
||||
startInfo.Environment.Remove(
|
||||
BakePublicationGuardPaths.NonceEnvironmentVariable);
|
||||
startInfo.Environment[
|
||||
BakePublicationGuardPaths.NonceEnvironmentVariable] =
|
||||
request.PublicationNonce
|
||||
?? throw new InvalidOperationException("publication nonce is missing");
|
||||
|
||||
using Process child = Process.Start(startInfo)
|
||||
?? throw new InvalidOperationException("orphan child did not start");
|
||||
File.WriteAllText(
|
||||
childPidPath,
|
||||
child.Id.ToString(System.Globalization.CultureInfo.InvariantCulture));
|
||||
await child.WaitForExitAsync(cancellationToken);
|
||||
if (child.ExitCode == 0)
|
||||
{
|
||||
long bytes = new FileInfo(request.OutputPath).Length;
|
||||
onStandardOutput("{\"v\":1,\"e\":\"started\",\"bakeToolVersion\":4}\n");
|
||||
onStandardOutput($"{{\"v\":1,\"e\":\"completed\","
|
||||
+ $"\"bakeToolVersion\":4,\"outputBytes\":{bytes},"
|
||||
+ "\"failures\":0}\n");
|
||||
}
|
||||
|
||||
return new BakeProcessResult(child.ExitCode, "orphan fixture child");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
using AcDream.Launcher.Core.Installation;
|
||||
using AcDream.Platform;
|
||||
|
||||
namespace AcDream.Launcher.Core.Tests.Installation;
|
||||
|
||||
public sealed class BakeProcessRunnerTests
|
||||
{
|
||||
[Fact]
|
||||
public void PublicationNonceIsEnvironmentOnlyAndVisibleArgumentsStayPinned()
|
||||
{
|
||||
string nonce = Guid.Parse("01234567-89ab-cdef-0123-456789abcdef")
|
||||
.ToString("N");
|
||||
var request = new BakeProcessRequest(
|
||||
"acdream-bake",
|
||||
"retail-dats",
|
||||
"data/pak/acdream.pak",
|
||||
7,
|
||||
nonce);
|
||||
|
||||
System.Diagnostics.ProcessStartInfo startInfo =
|
||||
SystemBakeProcessRunner.CreateStartInfo(request);
|
||||
|
||||
Assert.Equal(request.Arguments, startInfo.ArgumentList);
|
||||
Assert.DoesNotContain(nonce, startInfo.ArgumentList);
|
||||
Assert.Equal(
|
||||
nonce,
|
||||
startInfo.Environment[
|
||||
BakePublicationGuardPaths.NonceEnvironmentVariable]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void UnguardedRequestExplicitlyRemovesInheritedAuthorization()
|
||||
{
|
||||
var request = new BakeProcessRequest(
|
||||
"acdream-bake",
|
||||
"retail-dats",
|
||||
"data/pak/acdream.pak",
|
||||
1);
|
||||
|
||||
System.Diagnostics.ProcessStartInfo startInfo =
|
||||
SystemBakeProcessRunner.CreateStartInfo(request);
|
||||
|
||||
Assert.False(startInfo.Environment.ContainsKey(
|
||||
BakePublicationGuardPaths.NonceEnvironmentVariable));
|
||||
}
|
||||
}
|
||||
|
|
@ -85,6 +85,11 @@ public sealed class LauncherInstallerTests : IDisposable
|
|||
"--progress-json",
|
||||
],
|
||||
observedRequest.Arguments);
|
||||
Assert.True(BakePublicationGuardPaths.IsValidNonce(
|
||||
observedRequest.PublicationNonce));
|
||||
Assert.DoesNotContain(
|
||||
observedRequest.PublicationNonce!,
|
||||
observedRequest.Arguments);
|
||||
Assert.Equal(LauncherInstallRecordStore.CurrentBakeToolVersion,
|
||||
result.Record.BakeToolVersion);
|
||||
Assert.Equal(new FileInfo(result.Record.PreparedAssetPath).Length,
|
||||
|
|
@ -94,6 +99,9 @@ public sealed class LauncherInstallerTests : IDisposable
|
|||
result.Record.PreparedAssetSha256);
|
||||
Assert.Contains(progress, value => value.Phase == LauncherInstallPhase.BakingMeshes);
|
||||
Assert.Equal(LauncherInstallPhase.Completed, progress[^1].Phase);
|
||||
Assert.False(File.Exists(
|
||||
BakePublicationGuardPaths.GetAuthorizationPath(
|
||||
result.Record.PreparedAssetPath)));
|
||||
|
||||
var store = new LauncherInstallRecordStore(_paths);
|
||||
InstallRecordVerification verification = await store.LoadAndVerifyAsync();
|
||||
|
|
@ -434,6 +442,7 @@ public sealed class LauncherInstallerTests : IDisposable
|
|||
UseShellExecute = false,
|
||||
};
|
||||
startInfo.ArgumentList.Add(fixtureDll);
|
||||
startInfo.ArgumentList.Add("hold-install-lease");
|
||||
startInfo.ArgumentList.Add(
|
||||
InstallerTransactionLease.GetLockPath(store.DataDirectory));
|
||||
startInfo.ArgumentList.Add(staging);
|
||||
|
|
@ -479,6 +488,144 @@ public sealed class LauncherInstallerTests : IDisposable
|
|||
await File.ReadAllTextAsync(store.PreparedAssetPath));
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData("holds", 0)]
|
||||
[InlineData("late", 17)]
|
||||
public async Task OrphanBakeCanNeverPublishAfterRestartRecovery(
|
||||
string schedule,
|
||||
int expectedChildExitCode)
|
||||
{
|
||||
var store = new LauncherInstallRecordStore(_paths);
|
||||
LauncherInstallRecord old = await CreatePriorRecordAsync(store);
|
||||
string recordBefore = await File.ReadAllTextAsync(store.RecordPath);
|
||||
string control = Path.Combine(_root, "orphan-" + schedule);
|
||||
Directory.CreateDirectory(control);
|
||||
string ready = Path.Combine(control, "child-ready");
|
||||
string release = Path.Combine(control, "child-release");
|
||||
string childPid = Path.Combine(control, "child-pid");
|
||||
string childExit = Path.Combine(control, "child-exit");
|
||||
string fixtureDll = GetInstallLeaseFixturePath();
|
||||
var startInfo = new ProcessStartInfo("dotnet")
|
||||
{
|
||||
RedirectStandardError = true,
|
||||
RedirectStandardOutput = true,
|
||||
UseShellExecute = false,
|
||||
};
|
||||
foreach (string argument in new[]
|
||||
{
|
||||
fixtureDll,
|
||||
"orphan-parent",
|
||||
store.DataDirectory,
|
||||
_dats,
|
||||
_bakeExecutable,
|
||||
schedule,
|
||||
ready,
|
||||
release,
|
||||
childPid,
|
||||
childExit,
|
||||
})
|
||||
{
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
using Process parent = Process.Start(startInfo)
|
||||
?? throw new InvalidOperationException("Could not start orphan parent.");
|
||||
int orphanPid = 0;
|
||||
try
|
||||
{
|
||||
await WaitForFileAsync(ready, parent, TimeSpan.FromSeconds(15));
|
||||
orphanPid = int.Parse(
|
||||
await File.ReadAllTextAsync(childPid),
|
||||
System.Globalization.CultureInfo.InvariantCulture);
|
||||
Assert.True(File.Exists(
|
||||
LauncherInstallRecordStore.GetBackupPath(
|
||||
store.PreparedAssetPath)));
|
||||
Assert.True(File.Exists(
|
||||
BakePublicationGuardPaths.GetAuthorizationPath(
|
||||
store.PreparedAssetPath)));
|
||||
|
||||
parent.Kill(entireProcessTree: false);
|
||||
await parent.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
var restarted = new LauncherInstaller(
|
||||
_paths,
|
||||
_bakeExecutable,
|
||||
recordStore: new LauncherInstallRecordStore(_paths));
|
||||
Task<InstallRecordVerification> recovery =
|
||||
restarted.LoadExistingAsync();
|
||||
InstallRecordVerification recovered;
|
||||
if (schedule == "holds")
|
||||
{
|
||||
await Task.Delay(200);
|
||||
Assert.False(recovery.IsCompleted);
|
||||
File.WriteAllText(release, "release");
|
||||
recovered = await recovery.WaitAsync(TimeSpan.FromSeconds(15));
|
||||
}
|
||||
else
|
||||
{
|
||||
recovered = await recovery.WaitAsync(TimeSpan.FromSeconds(15));
|
||||
Assert.False(File.Exists(
|
||||
BakePublicationGuardPaths.GetAuthorizationPath(
|
||||
store.PreparedAssetPath)));
|
||||
File.WriteAllText(release, "release");
|
||||
}
|
||||
|
||||
Assert.True(recovered.IsVerified);
|
||||
Assert.Equal(old, recovered.Record);
|
||||
string canonicalAfterRecovery =
|
||||
await File.ReadAllTextAsync(store.PreparedAssetPath);
|
||||
string recordAfterRecovery =
|
||||
await File.ReadAllTextAsync(store.RecordPath);
|
||||
bool backupAfterRecovery = File.Exists(
|
||||
LauncherInstallRecordStore.GetBackupPath(
|
||||
store.PreparedAssetPath));
|
||||
|
||||
await WaitForFileAsync(childExit, TimeSpan.FromSeconds(15));
|
||||
Assert.Equal(
|
||||
expectedChildExitCode,
|
||||
int.Parse(
|
||||
await File.ReadAllTextAsync(childExit),
|
||||
System.Globalization.CultureInfo.InvariantCulture));
|
||||
if (schedule == "late")
|
||||
{
|
||||
Assert.Contains(
|
||||
"no longer authorized",
|
||||
await File.ReadAllTextAsync(childExit + ".error"),
|
||||
StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
await Task.Delay(200);
|
||||
|
||||
Assert.Equal(
|
||||
canonicalAfterRecovery,
|
||||
await File.ReadAllTextAsync(store.PreparedAssetPath));
|
||||
Assert.Equal("previous verified package", canonicalAfterRecovery);
|
||||
Assert.Equal(recordBefore, recordAfterRecovery);
|
||||
Assert.Equal(recordAfterRecovery, await File.ReadAllTextAsync(store.RecordPath));
|
||||
Assert.Equal(
|
||||
backupAfterRecovery,
|
||||
File.Exists(LauncherInstallRecordStore.GetBackupPath(
|
||||
store.PreparedAssetPath)));
|
||||
Assert.False(backupAfterRecovery);
|
||||
Assert.False(File.Exists(
|
||||
BakePublicationGuardPaths.GetAuthorizationPath(
|
||||
store.PreparedAssetPath)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
File.WriteAllText(release, "release");
|
||||
if (!parent.HasExited)
|
||||
{
|
||||
parent.Kill(entireProcessTree: false);
|
||||
await parent.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
|
||||
if (orphanPid != 0 && !File.Exists(childExit))
|
||||
{
|
||||
TryKill(orphanPid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<(
|
||||
LauncherInstaller Installer,
|
||||
LauncherInstallRecordStore Store,
|
||||
|
|
@ -548,6 +695,33 @@ public sealed class LauncherInstallerTests : IDisposable
|
|||
}
|
||||
}
|
||||
|
||||
private static async Task WaitForFileAsync(string path, TimeSpan timeout)
|
||||
{
|
||||
using var cancellation = new CancellationTokenSource(timeout);
|
||||
while (!File.Exists(path))
|
||||
{
|
||||
await Task.Delay(25, cancellation.Token);
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryKill(int processId)
|
||||
{
|
||||
try
|
||||
{
|
||||
using Process process = Process.GetProcessById(processId);
|
||||
if (!process.HasExited)
|
||||
{
|
||||
process.Kill(entireProcessTree: true);
|
||||
process.WaitForExit(5_000);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The orphan normally exits by itself; cleanup tolerates the
|
||||
// expected race with Process.GetProcessById.
|
||||
}
|
||||
}
|
||||
|
||||
private static string GetInstallLeaseFixturePath()
|
||||
{
|
||||
string root = FindRepositoryRoot();
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue