fix(launcher): harden updater crash recovery
This commit is contained in:
Erik
parent
2d2a5b5046
commit
1955ca8ab5
27 changed files with 2714 additions and 544 deletions
|
|
@ -8,16 +8,17 @@ public sealed record SelfUpdateStartupResult(
|
|||
string[] RemainingArguments);
|
||||
|
||||
/// <summary>
|
||||
/// Process-level rename dance for launcher self-update. Every child argument
|
||||
/// is passed through <see cref="ProcessStartInfo.ArgumentList"/> with
|
||||
/// <c>UseShellExecute=false</c>; no path or PID is ever interpolated into a
|
||||
/// shell command.
|
||||
/// Process-level self-update bootstrap. Every child argument is passed through
|
||||
/// <see cref="ProcessStartInfo.ArgumentList"/> with shell execution disabled.
|
||||
/// </summary>
|
||||
public static class LauncherSelfUpdateBootstrap
|
||||
{
|
||||
public const string HelperArgument = "--acdream-self-update-helper-v1";
|
||||
public const string ConfirmArgument = "--acdream-self-update-confirm-v1";
|
||||
internal const string DeferredArgument = "--acdream-self-update-deferred-v1";
|
||||
internal const int DeferredLeaseExitCode = 73;
|
||||
private static readonly TimeSpan ConfirmationTimeout = TimeSpan.FromSeconds(30);
|
||||
private static readonly TimeSpan CleanupTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
public static async Task<SelfUpdateStartupResult> HandleAsync(
|
||||
string[] args,
|
||||
|
|
@ -32,10 +33,16 @@ public static class LauncherSelfUpdateBootstrap
|
|||
Path.GetFullPath(launcherBaseDirectory));
|
||||
string executable = Path.GetFullPath(currentExecutablePath);
|
||||
|
||||
if (args.Length > 0
|
||||
&& string.Equals(args[0], DeferredArgument, StringComparison.Ordinal))
|
||||
{
|
||||
return new SelfUpdateStartupResult(false, 0, args[1..]);
|
||||
}
|
||||
|
||||
if (args.Length > 0
|
||||
&& string.Equals(args[0], HelperArgument, StringComparison.Ordinal))
|
||||
{
|
||||
if (args.Length != 4
|
||||
if (args.Length < 4
|
||||
|| !int.TryParse(
|
||||
args[1],
|
||||
System.Globalization.NumberStyles.None,
|
||||
|
|
@ -51,6 +58,7 @@ public static class LauncherSelfUpdateBootstrap
|
|||
parentPid,
|
||||
args[2],
|
||||
args[3],
|
||||
args[4..],
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return new SelfUpdateStartupResult(true, exitCode, []);
|
||||
|
|
@ -59,7 +67,7 @@ public static class LauncherSelfUpdateBootstrap
|
|||
if (args.Length > 0
|
||||
&& string.Equals(args[0], ConfirmArgument, StringComparison.Ordinal))
|
||||
{
|
||||
if (args.Length != 2)
|
||||
if (args.Length < 2)
|
||||
{
|
||||
return new SelfUpdateStartupResult(true, 64, []);
|
||||
}
|
||||
|
|
@ -70,82 +78,97 @@ public static class LauncherSelfUpdateBootstrap
|
|||
executable,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return new SelfUpdateStartupResult(false, 0, []);
|
||||
}
|
||||
|
||||
SelfUpdatePlan? plan = await manager.LoadPendingAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (plan is null)
|
||||
{
|
||||
return new SelfUpdateStartupResult(false, 0, args);
|
||||
}
|
||||
|
||||
if (!PathsEqual(plan.TargetDirectory, baseDirectory))
|
||||
{
|
||||
throw new LauncherUpdateException(
|
||||
"The pending self-update targets a different launcher directory.");
|
||||
}
|
||||
|
||||
if (plan.State == SelfUpdatePlanState.AwaitingConfirmation)
|
||||
{
|
||||
if (!manager.IsConfirmed(plan.TransactionId))
|
||||
{
|
||||
await manager.ConfirmAsync(
|
||||
plan.TransactionId,
|
||||
baseDirectory,
|
||||
executable,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await manager.CompleteConfirmedAsync(
|
||||
plan.TransactionId,
|
||||
await FinishConfirmedCleanupAsync(
|
||||
manager,
|
||||
baseDirectory,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return new SelfUpdateStartupResult(false, 0, args[2..]);
|
||||
}
|
||||
|
||||
if (!manager.Barrier.TryAcquireExclusive(
|
||||
out UpdateSessionBarrier.ExclusiveLease? startupLease))
|
||||
{
|
||||
// A running session or another launcher is staging. Reading the
|
||||
// plan is safe, but cleanup or starting a competing helper is not.
|
||||
return new SelfUpdateStartupResult(false, 0, args);
|
||||
}
|
||||
|
||||
string suffix = plan.Rid.StartsWith("win-", StringComparison.Ordinal)
|
||||
? ".exe"
|
||||
: string.Empty;
|
||||
string expectedExecutable = ClientVersionStore.ResolveContained(
|
||||
baseDirectory,
|
||||
"acdream-launcher" + suffix);
|
||||
if (!PathsEqual(executable, expectedExecutable))
|
||||
using (UpdateSessionBarrier.ExclusiveLease lease = startupLease
|
||||
?? throw new InvalidOperationException("Exclusive startup lease is missing."))
|
||||
{
|
||||
throw new LauncherUpdateException(
|
||||
"Self-update can start only from the published acdream-launcher executable.");
|
||||
}
|
||||
SelfUpdatePlan? plan = await manager.LoadPendingAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
_ = manager.CleanupOwnedResidueUnderLease(
|
||||
plan,
|
||||
baseDirectory,
|
||||
lease);
|
||||
if (plan is null)
|
||||
{
|
||||
return new SelfUpdateStartupResult(false, 0, args);
|
||||
}
|
||||
|
||||
string helperPath = manager.GetHelperPath(plan.TransactionId);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(helperPath)!);
|
||||
VerifiedArtifactDownloader.TryDelete(helperPath);
|
||||
File.Copy(executable, helperPath, overwrite: false);
|
||||
if (OperatingSystem.IsLinux())
|
||||
{
|
||||
File.SetUnixFileMode(
|
||||
helperPath,
|
||||
UnixFileMode.UserRead
|
||||
| UnixFileMode.UserWrite
|
||||
| UnixFileMode.UserExecute);
|
||||
}
|
||||
if (!PathsEqual(plan.TargetDirectory, baseDirectory))
|
||||
{
|
||||
throw new LauncherUpdateException(
|
||||
"The pending self-update targets a different launcher directory.");
|
||||
}
|
||||
|
||||
var startInfo = new ProcessStartInfo(helperPath)
|
||||
{
|
||||
UseShellExecute = false,
|
||||
WorkingDirectory = manager.GetTransactionDirectory(plan.TransactionId),
|
||||
};
|
||||
startInfo.ArgumentList.Add(HelperArgument);
|
||||
startInfo.ArgumentList.Add(
|
||||
Environment.ProcessId.ToString(
|
||||
System.Globalization.CultureInfo.InvariantCulture));
|
||||
startInfo.ArgumentList.Add(baseDirectory);
|
||||
startInfo.ArgumentList.Add(plan.TransactionId);
|
||||
_ = Process.Start(startInfo)
|
||||
?? throw new LauncherUpdateException(
|
||||
"The launcher self-update helper could not be started.");
|
||||
return new SelfUpdateStartupResult(true, 0, []);
|
||||
if (plan.State == SelfUpdatePlanState.AwaitingConfirmation)
|
||||
{
|
||||
if (!manager.IsConfirmed(plan.TransactionId))
|
||||
{
|
||||
await manager.ConfirmAsync(
|
||||
plan.TransactionId,
|
||||
baseDirectory,
|
||||
executable,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await manager.CompleteConfirmedAsync(
|
||||
plan.TransactionId,
|
||||
baseDirectory,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
_ = manager.CleanupOwnedResidueUnderLease(
|
||||
pending: null,
|
||||
baseDirectory,
|
||||
lease);
|
||||
return new SelfUpdateStartupResult(false, 0, args);
|
||||
}
|
||||
|
||||
string expectedExecutable = ClientVersionStore.ResolveContained(
|
||||
baseDirectory,
|
||||
GetLauncherFileName(plan.Rid));
|
||||
if (!PathsEqual(executable, expectedExecutable))
|
||||
{
|
||||
throw new LauncherUpdateException(
|
||||
"Self-update can start only from the published acdream-launcher executable.");
|
||||
}
|
||||
|
||||
string helperPath = manager.GetStagedLauncherPath(plan);
|
||||
var startInfo = new ProcessStartInfo(helperPath)
|
||||
{
|
||||
UseShellExecute = false,
|
||||
WorkingDirectory = manager.GetPayloadDirectory(plan.TransactionId),
|
||||
};
|
||||
startInfo.ArgumentList.Add(HelperArgument);
|
||||
startInfo.ArgumentList.Add(
|
||||
Environment.ProcessId.ToString(
|
||||
System.Globalization.CultureInfo.InvariantCulture));
|
||||
startInfo.ArgumentList.Add(baseDirectory);
|
||||
startInfo.ArgumentList.Add(plan.TransactionId);
|
||||
foreach (string argument in args)
|
||||
{
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
_ = Process.Start(startInfo)
|
||||
?? throw new LauncherUpdateException(
|
||||
"The launcher self-update helper could not be started.");
|
||||
return new SelfUpdateStartupResult(true, 0, []);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<int> RunHelperAsync(
|
||||
|
|
@ -153,6 +176,7 @@ public static class LauncherSelfUpdateBootstrap
|
|||
int parentPid,
|
||||
string targetDirectory,
|
||||
string transactionId,
|
||||
IReadOnlyList<string> publicArguments,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
SelfUpdatePlan plan = await manager.LoadPendingAsync(cancellationToken)
|
||||
|
|
@ -164,12 +188,15 @@ public static class LauncherSelfUpdateBootstrap
|
|||
"The helper transaction does not match the pending self-update.");
|
||||
}
|
||||
|
||||
string suffix = plan.Rid.StartsWith("win-", StringComparison.Ordinal)
|
||||
? ".exe"
|
||||
: string.Empty;
|
||||
if (!PathsEqual(plan.TargetDirectory, targetDirectory))
|
||||
{
|
||||
throw new LauncherUpdateException(
|
||||
"The helper target does not match the pending self-update.");
|
||||
}
|
||||
|
||||
string launcherPath = ClientVersionStore.ResolveContained(
|
||||
targetDirectory,
|
||||
"acdream-launcher" + suffix);
|
||||
GetLauncherFileName(plan.Rid));
|
||||
var startInfo = new ProcessStartInfo(launcherPath)
|
||||
{
|
||||
UseShellExecute = false,
|
||||
|
|
@ -177,97 +204,184 @@ public static class LauncherSelfUpdateBootstrap
|
|||
};
|
||||
startInfo.ArgumentList.Add(ConfirmArgument);
|
||||
startInfo.ArgumentList.Add(transactionId);
|
||||
|
||||
Process? replacement = null;
|
||||
UpdateSessionBarrier.ExclusiveLease? updateLease = null;
|
||||
bool appliedByThisHelper = false;
|
||||
try
|
||||
foreach (string argument in publicArguments)
|
||||
{
|
||||
await WaitForParentExitAsync(parentPid, cancellationToken).ConfigureAwait(false);
|
||||
updateLease = manager.Barrier.AcquireExclusive();
|
||||
plan = await manager.ApplyPendingAsync(targetDirectory, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
appliedByThisHelper = true;
|
||||
replacement = Process.Start(startInfo)
|
||||
?? throw new LauncherUpdateException(
|
||||
"The updated launcher could not be started.");
|
||||
DateTimeOffset deadline = DateTimeOffset.UtcNow + ConfirmationTimeout;
|
||||
while (!manager.IsConfirmed(transactionId))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (replacement.HasExited || DateTimeOffset.UtcNow >= deadline)
|
||||
{
|
||||
throw new LauncherUpdateException(
|
||||
replacement.HasExited
|
||||
? $"The updated launcher exited with code {replacement.ExitCode} "
|
||||
+ "before confirming startup."
|
||||
: "The updated launcher did not confirm startup in time.");
|
||||
}
|
||||
|
||||
await Task.Delay(100, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await manager.CompleteConfirmedAsync(
|
||||
transactionId,
|
||||
targetDirectory,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return 0;
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
catch
|
||||
|
||||
await WaitForParentExitAsync(parentPid, cancellationToken).ConfigureAwait(false);
|
||||
if (!manager.Barrier.TryAcquireExclusive(
|
||||
out UpdateSessionBarrier.ExclusiveLease? updateLease))
|
||||
{
|
||||
if (replacement is { HasExited: false })
|
||||
// Do not restart the canonical launcher: it would immediately see
|
||||
// the same staged plan and create an unbounded helper loop.
|
||||
return DeferredLeaseExitCode;
|
||||
}
|
||||
|
||||
using (UpdateSessionBarrier.ExclusiveLease lease = updateLease
|
||||
?? throw new InvalidOperationException("Exclusive update lease is missing."))
|
||||
{
|
||||
plan = await manager.LoadPendingAsync(cancellationToken)
|
||||
.ConfigureAwait(false)
|
||||
?? throw new LauncherUpdateException(
|
||||
"The helper found no pending self-update after acquiring the lease.");
|
||||
if (!string.Equals(
|
||||
plan.TransactionId,
|
||||
transactionId,
|
||||
StringComparison.Ordinal)
|
||||
|| !PathsEqual(plan.TargetDirectory, targetDirectory))
|
||||
{
|
||||
replacement.Kill(entireProcessTree: true);
|
||||
await replacement.WaitForExitAsync(CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
throw new LauncherUpdateException(
|
||||
"The pending self-update changed before the helper acquired its lease.");
|
||||
}
|
||||
|
||||
_ = manager.CleanupOwnedResidueUnderLease(
|
||||
plan,
|
||||
targetDirectory,
|
||||
lease);
|
||||
Process? replacement = null;
|
||||
bool appliedByThisHelper = false;
|
||||
try
|
||||
{
|
||||
if (appliedByThisHelper)
|
||||
plan = await manager.ApplyPendingAsync(targetDirectory, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
appliedByThisHelper = true;
|
||||
replacement = Process.Start(startInfo)
|
||||
?? throw new LauncherUpdateException(
|
||||
"The updated launcher could not be started.");
|
||||
DateTimeOffset deadline = DateTimeOffset.UtcNow + ConfirmationTimeout;
|
||||
while (!manager.IsConfirmed(transactionId))
|
||||
{
|
||||
SelfUpdatePlan? pending = await manager.LoadPendingAsync(
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
if (pending?.State == SelfUpdatePlanState.Applying)
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (replacement.HasExited || DateTimeOffset.UtcNow >= deadline)
|
||||
{
|
||||
_ = await manager.RecoverApplyingAsync(
|
||||
targetDirectory,
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else if (pending?.State == SelfUpdatePlanState.AwaitingConfirmation)
|
||||
{
|
||||
_ = await manager.RollbackAwaitingConfirmationAsync(
|
||||
targetDirectory,
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
throw new LauncherUpdateException(
|
||||
replacement.HasExited
|
||||
? $"The updated launcher exited with code {replacement.ExitCode} "
|
||||
+ "before confirming startup."
|
||||
: "The updated launcher did not confirm startup in time.");
|
||||
}
|
||||
|
||||
await Task.Delay(100, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await manager.CompleteConfirmedAsync(
|
||||
transactionId,
|
||||
targetDirectory,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
return 0;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Do not start an executable from an ambiguous half-applied
|
||||
// state. A subsequent startup replays the durable journal.
|
||||
return 75;
|
||||
}
|
||||
if (replacement is { HasExited: false })
|
||||
{
|
||||
replacement.Kill(entireProcessTree: true);
|
||||
await replacement.WaitForExitAsync(CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var restored = new ProcessStartInfo(launcherPath)
|
||||
try
|
||||
{
|
||||
if (appliedByThisHelper)
|
||||
{
|
||||
SelfUpdatePlan? pending = await manager.LoadPendingAsync(
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
if (pending?.State == SelfUpdatePlanState.Applying)
|
||||
{
|
||||
_ = await manager.RecoverApplyingAsync(
|
||||
targetDirectory,
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
else if (pending?.State == SelfUpdatePlanState.AwaitingConfirmation)
|
||||
{
|
||||
_ = await manager.RollbackAwaitingConfirmationAsync(
|
||||
targetDirectory,
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// An ambiguous state must not start either executable.
|
||||
return 75;
|
||||
}
|
||||
|
||||
var restored = new ProcessStartInfo(launcherPath)
|
||||
{
|
||||
UseShellExecute = false,
|
||||
WorkingDirectory = Path.GetFullPath(targetDirectory),
|
||||
};
|
||||
restored.ArgumentList.Add(DeferredArgument);
|
||||
foreach (string argument in publicArguments)
|
||||
{
|
||||
restored.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
_ = Process.Start(restored);
|
||||
return 74;
|
||||
}
|
||||
finally
|
||||
{
|
||||
UseShellExecute = false,
|
||||
WorkingDirectory = Path.GetFullPath(targetDirectory),
|
||||
};
|
||||
_ = Process.Start(restored);
|
||||
return 74;
|
||||
}
|
||||
finally
|
||||
{
|
||||
replacement?.Dispose();
|
||||
updateLease?.Dispose();
|
||||
replacement?.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task FinishConfirmedCleanupAsync(
|
||||
LauncherSelfUpdateManager manager,
|
||||
string targetDirectory,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
DateTimeOffset deadline = DateTimeOffset.UtcNow + CleanupTimeout;
|
||||
do
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (manager.Barrier.TryAcquireExclusive(
|
||||
out UpdateSessionBarrier.ExclusiveLease? lease))
|
||||
{
|
||||
using (UpdateSessionBarrier.ExclusiveLease acquiredLease = lease
|
||||
?? throw new InvalidOperationException(
|
||||
"Exclusive cleanup lease is missing."))
|
||||
{
|
||||
SelfUpdatePlan? pending = await manager.LoadPendingAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
if (pending is
|
||||
{
|
||||
State: SelfUpdatePlanState.AwaitingConfirmation,
|
||||
}
|
||||
&& manager.IsConfirmed(pending.TransactionId))
|
||||
{
|
||||
await manager.CompleteConfirmedAsync(
|
||||
pending.TransactionId,
|
||||
targetDirectory,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
pending = null;
|
||||
}
|
||||
|
||||
if (manager.CleanupOwnedResidueUnderLease(
|
||||
pending,
|
||||
targetDirectory,
|
||||
acquiredLease))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await Task.Delay(50, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
while (DateTimeOffset.UtcNow < deadline);
|
||||
}
|
||||
|
||||
private static string GetLauncherFileName(string rid) =>
|
||||
"acdream-launcher"
|
||||
+ (rid.StartsWith("win-", StringComparison.Ordinal) ? ".exe" : string.Empty);
|
||||
|
||||
private static async Task WaitForParentExitAsync(
|
||||
int parentPid,
|
||||
CancellationToken cancellationToken)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue