fix(launcher): harden updater crash recovery
This commit is contained in:
Erik
parent
2d2a5b5046
commit
1955ca8ab5
27 changed files with 2714 additions and 544 deletions
|
|
@ -0,0 +1,453 @@
|
|||
using System.Diagnostics;
|
||||
using AcDream.Launcher.Core.Integrity;
|
||||
using AcDream.Launcher.Core.Updates;
|
||||
|
||||
namespace AcDream.Launcher.Core.Tests.Updates;
|
||||
|
||||
public sealed class LauncherSelfUpdateProcessTests : IDisposable
|
||||
{
|
||||
private const string FixtureBaseName =
|
||||
"AcDream.Launcher.Core.Tests.Fixtures.InstallLeaseHolder";
|
||||
private const string DataEnvironment = "ACDREAM_SELF_UPDATE_FIXTURE_DATA";
|
||||
private const string TargetEnvironment = "ACDREAM_SELF_UPDATE_FIXTURE_TARGET";
|
||||
private const string HelperPidEnvironment =
|
||||
"ACDREAM_SELF_UPDATE_FIXTURE_HELPER_PID";
|
||||
|
||||
private readonly string _root = Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"acdream-self-update-process-tests",
|
||||
Guid.NewGuid().ToString("N"));
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (Directory.Exists(_root))
|
||||
{
|
||||
Directory.Delete(_root, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task KilledAfterCanonicalReplaceCanInvokeCanonicalAndConvergeAutomatically()
|
||||
{
|
||||
string data = Path.Combine(_root, "data");
|
||||
string target = Path.Combine(_root, "launcher");
|
||||
string ready = Path.Combine(_root, "crash.ready");
|
||||
string launched = Path.Combine(_root, "replacement.ready");
|
||||
string helperPidPath = Path.Combine(_root, "helper.pid");
|
||||
Directory.CreateDirectory(_root);
|
||||
string rid = LauncherRuntimeIdentity.DetectRid();
|
||||
PreparedLauncher prepared = PrepareLauncherClosure(target, rid);
|
||||
string oldHash = await FileIntegrity.ComputeSha256HexAsync(prepared.CanonicalPath);
|
||||
using var server = new LocalHttpFixture();
|
||||
server.Add("launcher.zip", prepared.NewArchive);
|
||||
using var http = new HttpClient();
|
||||
var manager = new LauncherSelfUpdateManager(UpdateTestData.Paths(_root), http);
|
||||
SelfUpdateStageResult staged = await manager.StageAsync(
|
||||
LauncherVersion.Parse("2.0.0"),
|
||||
rid,
|
||||
new ReleaseArtifact(
|
||||
server.UriFor("launcher.zip"),
|
||||
UpdateTestData.Sha256(prepared.NewArchive),
|
||||
prepared.NewArchive.LongLength),
|
||||
target,
|
||||
progress: null,
|
||||
CancellationToken.None);
|
||||
SelfUpdatePlan plan = Assert.IsType<SelfUpdatePlan>(await manager.LoadPendingAsync());
|
||||
|
||||
using Process crash = StartFixture(
|
||||
["crash-self-update", data, target, ready, prepared.CanonicalName]);
|
||||
try
|
||||
{
|
||||
await WaitForFileAsync(ready, crash, TimeSpan.FromSeconds(20));
|
||||
Assert.True(File.Exists(prepared.CanonicalPath));
|
||||
crash.Kill(entireProcessTree: true);
|
||||
await crash.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.True(File.Exists(prepared.CanonicalPath));
|
||||
string boundaryHash = await FileIntegrity.ComputeSha256HexAsync(
|
||||
prepared.CanonicalPath);
|
||||
Assert.Contains(boundaryHash, new[] { oldHash, prepared.NewCanonicalHash });
|
||||
|
||||
var environment = new Dictionary<string, string>
|
||||
{
|
||||
[DataEnvironment] = data,
|
||||
[TargetEnvironment] = target,
|
||||
[HelperPidEnvironment] = helperPidPath,
|
||||
};
|
||||
using Process canonical = StartProcess(
|
||||
prepared.CanonicalPath,
|
||||
["canonical-probe", launched],
|
||||
environment);
|
||||
await canonical.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(20));
|
||||
Assert.Equal(0, canonical.ExitCode);
|
||||
await WaitForFileAsync(launched, process: null, TimeSpan.FromSeconds(30));
|
||||
await WaitUntilAsync(
|
||||
() => !File.Exists(manager.PendingPlanPath),
|
||||
TimeSpan.FromSeconds(30),
|
||||
"The self-update journal did not converge.");
|
||||
|
||||
Assert.Equal(
|
||||
prepared.NewCanonicalHash,
|
||||
await FileIntegrity.ComputeSha256HexAsync(prepared.CanonicalPath));
|
||||
Assert.True(File.Exists(Path.Combine(
|
||||
target,
|
||||
LauncherSelfUpdateManager.InstallRecordFileName)));
|
||||
Assert.False(Directory.Exists(manager.GetTransactionDirectory(
|
||||
plan.TransactionId)));
|
||||
Assert.Empty(Directory.EnumerateDirectories(
|
||||
target,
|
||||
".acdream-self-update-*",
|
||||
SearchOption.TopDirectoryOnly));
|
||||
Assert.Null(await manager.LoadPendingAsync());
|
||||
|
||||
int replacementPid = ParsePid(await File.ReadAllTextAsync(launched));
|
||||
int helperPid = int.Parse(
|
||||
await File.ReadAllTextAsync(helperPidPath),
|
||||
System.Globalization.CultureInfo.InvariantCulture);
|
||||
await WaitForProcessExitAsync(replacementPid, TimeSpan.FromSeconds(10));
|
||||
await WaitForProcessExitAsync(helperPid, TimeSpan.FromSeconds(10));
|
||||
if (OperatingSystem.IsLinux())
|
||||
{
|
||||
Assert.True(
|
||||
(File.GetUnixFileMode(prepared.CanonicalPath)
|
||||
& (UnixFileMode.UserExecute
|
||||
| UnixFileMode.GroupExecute
|
||||
| UnixFileMode.OtherExecute)) != 0);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!crash.HasExited)
|
||||
{
|
||||
crash.Kill(entireProcessTree: true);
|
||||
await crash.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ConcurrentStartupCannotDeleteAVisibleSlowStageTransaction()
|
||||
{
|
||||
string data = Path.Combine(_root, "data");
|
||||
string target = Path.Combine(_root, "launcher");
|
||||
string resultPath = Path.Combine(_root, "startup.result");
|
||||
Directory.CreateDirectory(target);
|
||||
string rid = LauncherRuntimeIdentity.DetectRid();
|
||||
string canonicalName = LauncherName(rid);
|
||||
string canonical = Path.Combine(target, canonicalName);
|
||||
await File.WriteAllTextAsync(canonical, "running launcher");
|
||||
byte[] archive = UpdateTestData.LauncherZip(
|
||||
rid,
|
||||
new string('s', 2 * 1024 * 1024));
|
||||
using var server = new LocalHttpFixture();
|
||||
server.Add(
|
||||
"slow-launcher.zip",
|
||||
archive,
|
||||
chunkSize: 4096,
|
||||
chunkDelay: TimeSpan.FromMilliseconds(2));
|
||||
var observer = new LauncherSelfUpdateManager(
|
||||
UpdateTestData.Paths(_root),
|
||||
new HttpClient());
|
||||
using Process staging = StartFixture(
|
||||
[
|
||||
"stage-self-update",
|
||||
data,
|
||||
target,
|
||||
"2.0.0",
|
||||
rid,
|
||||
server.UriFor("slow-launcher.zip").AbsoluteUri,
|
||||
UpdateTestData.Sha256(archive),
|
||||
archive.LongLength.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
]);
|
||||
try
|
||||
{
|
||||
await WaitUntilAsync(
|
||||
() => Directory.Exists(observer.TransactionsDirectory)
|
||||
&& Directory.EnumerateDirectories(observer.TransactionsDirectory).Any(),
|
||||
TimeSpan.FromSeconds(10),
|
||||
"The slow staging transaction did not become visible.",
|
||||
staging);
|
||||
Assert.False(File.Exists(observer.PendingPlanPath));
|
||||
string transaction = Assert.Single(
|
||||
Directory.EnumerateDirectories(observer.TransactionsDirectory));
|
||||
|
||||
using Process startup = StartFixture(
|
||||
["bootstrap-probe", data, target, canonical, resultPath]);
|
||||
await startup.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.Equal(0, startup.ExitCode);
|
||||
Assert.Equal("ordinary", await File.ReadAllTextAsync(resultPath));
|
||||
Assert.True(Directory.Exists(transaction));
|
||||
Assert.False(File.Exists(observer.PendingPlanPath));
|
||||
|
||||
await staging.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(30));
|
||||
Assert.Equal(0, staging.ExitCode);
|
||||
SelfUpdatePlan plan = Assert.IsType<SelfUpdatePlan>(
|
||||
await observer.LoadPendingAsync());
|
||||
Assert.Equal(SelfUpdatePlanState.Staged, plan.State);
|
||||
Assert.True(Directory.Exists(transaction));
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (!staging.HasExited)
|
||||
{
|
||||
staging.Kill(entireProcessTree: true);
|
||||
await staging.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HelperDefersWithoutRestartWhenSharedSessionLeaseAppears()
|
||||
{
|
||||
string data = Path.Combine(_root, "data");
|
||||
string target = Path.Combine(_root, "launcher");
|
||||
string unexpectedLaunch = Path.Combine(_root, "unexpected-launch");
|
||||
string helperPid = Path.Combine(_root, "helper.pid");
|
||||
Directory.CreateDirectory(target);
|
||||
string rid = LauncherRuntimeIdentity.DetectRid();
|
||||
string canonical = Path.Combine(target, LauncherName(rid));
|
||||
await File.WriteAllTextAsync(canonical, "old-launcher");
|
||||
byte[] archive = UpdateTestData.LauncherZip(rid, "new-launcher");
|
||||
using var server = new LocalHttpFixture();
|
||||
server.Add("launcher.zip", archive);
|
||||
using var http = new HttpClient();
|
||||
var manager = new LauncherSelfUpdateManager(UpdateTestData.Paths(_root), http);
|
||||
_ = await manager.StageAsync(
|
||||
LauncherVersion.Parse("2.0.0"),
|
||||
rid,
|
||||
new ReleaseArtifact(
|
||||
server.UriFor("launcher.zip"),
|
||||
UpdateTestData.Sha256(archive),
|
||||
archive.LongLength),
|
||||
target,
|
||||
progress: null,
|
||||
CancellationToken.None);
|
||||
SelfUpdatePlan plan = Assert.IsType<SelfUpdatePlan>(await manager.LoadPendingAsync());
|
||||
using UpdateSessionBarrier.SessionLease session = manager.Barrier.AcquireSession();
|
||||
var environment = new Dictionary<string, string>
|
||||
{
|
||||
[DataEnvironment] = data,
|
||||
[TargetEnvironment] = target,
|
||||
[HelperPidEnvironment] = helperPid,
|
||||
};
|
||||
|
||||
using Process helper = StartFixture(
|
||||
[
|
||||
LauncherSelfUpdateBootstrap.HelperArgument,
|
||||
int.MaxValue.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
target,
|
||||
plan.TransactionId,
|
||||
"canonical-probe",
|
||||
unexpectedLaunch,
|
||||
], environment);
|
||||
await helper.WaitForExitAsync().WaitAsync(TimeSpan.FromSeconds(10));
|
||||
|
||||
Assert.Equal(LauncherSelfUpdateBootstrap.DeferredLeaseExitCode, helper.ExitCode);
|
||||
Assert.True(File.Exists(helperPid));
|
||||
Assert.False(File.Exists(unexpectedLaunch));
|
||||
Assert.Equal("old-launcher", await File.ReadAllTextAsync(canonical));
|
||||
SelfUpdatePlan deferred = Assert.IsType<SelfUpdatePlan>(
|
||||
await manager.LoadPendingAsync());
|
||||
Assert.Equal(SelfUpdatePlanState.Staged, deferred.State);
|
||||
Assert.Equal(plan.TransactionId, deferred.TransactionId);
|
||||
}
|
||||
|
||||
private PreparedLauncher PrepareLauncherClosure(string target, string rid)
|
||||
{
|
||||
string fixtureDirectory = GetFixtureDirectory();
|
||||
string fixtureAppHost = Path.Combine(
|
||||
fixtureDirectory,
|
||||
FixtureBaseName + (OperatingSystem.IsWindows() ? ".exe" : string.Empty));
|
||||
Assert.True(File.Exists(fixtureAppHost), $"Missing fixture apphost: {fixtureAppHost}");
|
||||
Directory.CreateDirectory(target);
|
||||
string canonicalName = LauncherName(rid);
|
||||
string canonicalPath = Path.Combine(target, canonicalName);
|
||||
var archiveEntries = new List<(string Name, byte[] Content, int? UnixAttributes)>();
|
||||
foreach (string source in Directory.EnumerateFiles(
|
||||
fixtureDirectory,
|
||||
"*",
|
||||
SearchOption.TopDirectoryOnly))
|
||||
{
|
||||
string sourceName = Path.GetFileName(source);
|
||||
bool isAppHost = PathsEqual(source, fixtureAppHost);
|
||||
string targetName = isAppHost ? canonicalName : sourceName;
|
||||
byte[] oldContent = File.ReadAllBytes(source);
|
||||
byte[] newContent = oldContent;
|
||||
if (isAppHost)
|
||||
{
|
||||
oldContent = [.. oldContent, .. "-old"u8.ToArray()];
|
||||
newContent = [.. newContent, .. "-new"u8.ToArray()];
|
||||
}
|
||||
|
||||
string targetPath = Path.Combine(target, targetName);
|
||||
File.WriteAllBytes(targetPath, oldContent);
|
||||
int unixAttributes = 0x81A4;
|
||||
if (OperatingSystem.IsLinux())
|
||||
{
|
||||
UnixFileMode mode = File.GetUnixFileMode(source);
|
||||
if (isAppHost)
|
||||
{
|
||||
mode |= UnixFileMode.UserExecute;
|
||||
}
|
||||
|
||||
File.SetUnixFileMode(targetPath, mode);
|
||||
unixAttributes = 0x8000 | (int)mode;
|
||||
}
|
||||
else if (isAppHost)
|
||||
{
|
||||
unixAttributes = 0x81ED;
|
||||
}
|
||||
|
||||
archiveEntries.Add((targetName, newContent, unixAttributes));
|
||||
}
|
||||
|
||||
byte[] archive = UpdateTestData.CreateZip(archiveEntries);
|
||||
byte[] newCanonical = Assert.Single(
|
||||
archiveEntries,
|
||||
entry => entry.Name == canonicalName).Content;
|
||||
return new PreparedLauncher(
|
||||
canonicalName,
|
||||
canonicalPath,
|
||||
archive,
|
||||
UpdateTestData.Sha256(newCanonical));
|
||||
}
|
||||
|
||||
private static Process StartFixture(
|
||||
IReadOnlyList<string> arguments,
|
||||
IReadOnlyDictionary<string, string>? environment = null) =>
|
||||
StartProcess("dotnet", [GetFixtureDllPath(), .. arguments], environment);
|
||||
|
||||
private static Process StartProcess(
|
||||
string executable,
|
||||
IReadOnlyList<string> arguments,
|
||||
IReadOnlyDictionary<string, string>? environment = null)
|
||||
{
|
||||
var start = new ProcessStartInfo(executable)
|
||||
{
|
||||
UseShellExecute = false,
|
||||
RedirectStandardError = true,
|
||||
RedirectStandardOutput = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
foreach (string argument in arguments)
|
||||
{
|
||||
start.ArgumentList.Add(argument);
|
||||
}
|
||||
|
||||
if (environment is not null)
|
||||
{
|
||||
foreach ((string name, string value) in environment)
|
||||
{
|
||||
start.Environment[name] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return Process.Start(start)
|
||||
?? throw new InvalidOperationException($"Could not start '{executable}'.");
|
||||
}
|
||||
|
||||
private static async Task WaitForFileAsync(
|
||||
string path,
|
||||
Process? process,
|
||||
TimeSpan timeout) =>
|
||||
await WaitUntilAsync(
|
||||
() => File.Exists(path),
|
||||
timeout,
|
||||
$"Timed out waiting for '{path}'.",
|
||||
process);
|
||||
|
||||
private static async Task WaitUntilAsync(
|
||||
Func<bool> condition,
|
||||
TimeSpan timeout,
|
||||
string failure,
|
||||
Process? process = null)
|
||||
{
|
||||
DateTimeOffset deadline = DateTimeOffset.UtcNow + timeout;
|
||||
while (!condition())
|
||||
{
|
||||
if (process?.HasExited == true)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"{failure} Process exited {process.ExitCode}. stdout: "
|
||||
+ await process.StandardOutput.ReadToEndAsync()
|
||||
+ " stderr: "
|
||||
+ await process.StandardError.ReadToEndAsync());
|
||||
}
|
||||
|
||||
if (DateTimeOffset.UtcNow >= deadline)
|
||||
{
|
||||
throw new TimeoutException(failure);
|
||||
}
|
||||
|
||||
await Task.Delay(20);
|
||||
}
|
||||
}
|
||||
|
||||
private static int ParsePid(string marker)
|
||||
{
|
||||
string value = marker.Split('|', 2)[0];
|
||||
return int.Parse(value, System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static async Task WaitForProcessExitAsync(int pid, TimeSpan timeout)
|
||||
{
|
||||
try
|
||||
{
|
||||
using Process process = Process.GetProcessById(pid);
|
||||
await process.WaitForExitAsync().WaitAsync(timeout);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
// It exited before the test opened the process handle.
|
||||
}
|
||||
}
|
||||
|
||||
private static string LauncherName(string rid) =>
|
||||
"acdream-launcher"
|
||||
+ (rid.StartsWith("win-", StringComparison.Ordinal) ? ".exe" : string.Empty);
|
||||
|
||||
private static string GetFixtureDllPath() =>
|
||||
Path.Combine(GetFixtureDirectory(), FixtureBaseName + ".dll");
|
||||
|
||||
private static string GetFixtureDirectory()
|
||||
{
|
||||
string configuration = new DirectoryInfo(AppContext.BaseDirectory)
|
||||
.Parent?.Name ?? "Release";
|
||||
return Path.Combine(
|
||||
FindRepositoryRoot(),
|
||||
"tests",
|
||||
"AcDream.Launcher.Core.Tests.Fixtures.InstallLeaseHolder",
|
||||
"bin",
|
||||
configuration,
|
||||
"net10.0");
|
||||
}
|
||||
|
||||
private static string FindRepositoryRoot()
|
||||
{
|
||||
for (var directory = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
directory is not null;
|
||||
directory = directory.Parent)
|
||||
{
|
||||
if (File.Exists(Path.Combine(directory.FullName, "AcDream.slnx")))
|
||||
{
|
||||
return directory.FullName;
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Repository root was not found.");
|
||||
}
|
||||
|
||||
private static bool PathsEqual(string left, string right) => string.Equals(
|
||||
Path.GetFullPath(left),
|
||||
Path.GetFullPath(right),
|
||||
OperatingSystem.IsWindows()
|
||||
? StringComparison.OrdinalIgnoreCase
|
||||
: StringComparison.Ordinal);
|
||||
|
||||
private sealed record PreparedLauncher(
|
||||
string CanonicalName,
|
||||
string CanonicalPath,
|
||||
byte[] NewArchive,
|
||||
string NewCanonicalHash);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue