fix(launcher): guard orphan bake publication
This commit is contained in:
parent
3f68895120
commit
208a70ac83
14 changed files with 829 additions and 55 deletions
|
|
@ -12,6 +12,7 @@
|
|||
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="AcDream.Bake.Tests" />
|
||||
<InternalsVisibleTo Include="AcDream.Launcher.Core.Tests.Fixtures.InstallLeaseHolder" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
|
@ -24,6 +25,7 @@
|
|||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\AcDream.Content\AcDream.Content.csproj" />
|
||||
<ProjectReference Include="..\AcDream.Platform\AcDream.Platform.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -16,6 +16,21 @@ public static class BakeOutputTransaction
|
|||
Func<string, TResult> writeTemporary,
|
||||
Action<string, TResult> validateTemporary,
|
||||
CancellationToken cancellationToken = default)
|
||||
=> WriteValidateAndPublish(
|
||||
destinationPath,
|
||||
writeTemporary,
|
||||
validateTemporary,
|
||||
beforePublicationLock: null,
|
||||
beforePromotion: null,
|
||||
cancellationToken);
|
||||
|
||||
internal static TResult WriteValidateAndPublish<TResult>(
|
||||
string destinationPath,
|
||||
Func<string, TResult> writeTemporary,
|
||||
Action<string, TResult> validateTemporary,
|
||||
Action? beforePublicationLock,
|
||||
Action? beforePromotion,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(destinationPath);
|
||||
ArgumentNullException.ThrowIfNull(writeTemporary);
|
||||
|
|
@ -36,6 +51,14 @@ public static class BakeOutputTransaction
|
|||
cancellationToken.ThrowIfCancellationRequested();
|
||||
validateTemporary(temporaryPath, result);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
beforePublicationLock?.Invoke();
|
||||
using IDisposable? publication =
|
||||
BakePublicationGuard.AcquireIfRequested(
|
||||
fullDestination,
|
||||
cancellationToken);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
beforePromotion?.Invoke();
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
// Same-volume MoveFileEx/rename is the publication primitive.
|
||||
// File.Replace additionally performs destination metadata/backup
|
||||
|
|
|
|||
80
src/AcDream.Bake/BakePublicationGuard.cs
Normal file
80
src/AcDream.Bake/BakePublicationGuard.cs
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
using AcDream.Platform;
|
||||
|
||||
namespace AcDream.Bake;
|
||||
|
||||
/// <summary>
|
||||
/// Optional launcher authorization checked immediately before atomic
|
||||
/// publication. Standalone Bake runs have no nonce environment variable and
|
||||
/// retain the original unguarded behavior.
|
||||
/// </summary>
|
||||
internal static class BakePublicationGuard
|
||||
{
|
||||
private static readonly TimeSpan RetryDelay = TimeSpan.FromMilliseconds(50);
|
||||
|
||||
internal static IDisposable? AcquireIfRequested(
|
||||
string outputPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string? nonce = Environment.GetEnvironmentVariable(
|
||||
BakePublicationGuardPaths.NonceEnvironmentVariable);
|
||||
if (nonce is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!BakePublicationGuardPaths.IsValidNonce(nonce))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"The launcher bake publication nonce is invalid.");
|
||||
}
|
||||
|
||||
string lockPath = BakePublicationGuardPaths.GetPublishLockPath(
|
||||
outputPath);
|
||||
Directory.CreateDirectory(
|
||||
Path.GetDirectoryName(lockPath)
|
||||
?? throw new InvalidOperationException(
|
||||
"The bake publication lock has no parent directory."));
|
||||
|
||||
FileStream? lease = null;
|
||||
while (lease is null)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
lease = new FileStream(
|
||||
lockPath,
|
||||
FileMode.OpenOrCreate,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None,
|
||||
bufferSize: 1,
|
||||
options: FileOptions.None);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
cancellationToken.WaitHandle.WaitOne(RetryDelay);
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
string authorizationPath =
|
||||
BakePublicationGuardPaths.GetAuthorizationPath(outputPath);
|
||||
string authorized = File.Exists(authorizationPath)
|
||||
? File.ReadAllText(authorizationPath)
|
||||
: string.Empty;
|
||||
if (!string.Equals(authorized, nonce, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"This bake process is no longer authorized to publish its output.");
|
||||
}
|
||||
|
||||
return lease;
|
||||
}
|
||||
catch
|
||||
{
|
||||
lease.Dispose();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -14,6 +14,7 @@
|
|||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<InternalsVisibleTo Include="AcDream.Launcher.Core.Tests" />
|
||||
<InternalsVisibleTo Include="AcDream.Launcher.Core.Tests.Fixtures.InstallLeaseHolder" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\AcDream.Platform\AcDream.Platform.csproj" />
|
||||
|
|
|
|||
|
|
@ -51,14 +51,22 @@ internal static class BakeOutputStagingContract
|
|||
}
|
||||
|
||||
string destinationFileName = Path.GetFileName(fullDestination);
|
||||
foreach (string candidate in Directory.EnumerateFiles(directory))
|
||||
try
|
||||
{
|
||||
if (IsOwnedStagingFileName(
|
||||
Path.GetFileName(candidate),
|
||||
destinationFileName))
|
||||
foreach (string candidate in Directory.EnumerateFiles(directory))
|
||||
{
|
||||
LauncherInstallRecordStore.TryDelete(candidate);
|
||||
if (IsOwnedStagingFileName(
|
||||
Path.GetFileName(candidate),
|
||||
destinationFileName))
|
||||
{
|
||||
LauncherInstallRecordStore.TryDelete(candidate);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
// Best effort: these files are never launchable. A later startup
|
||||
// retries exact-name cleanup under the publication lock.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using AcDream.Platform;
|
||||
|
||||
namespace AcDream.Launcher.Core.Installation;
|
||||
|
||||
|
|
@ -9,7 +10,8 @@ public sealed record BakeProcessRequest(
|
|||
string ExecutablePath,
|
||||
string DatDirectory,
|
||||
string OutputPath,
|
||||
int Threads)
|
||||
int Threads,
|
||||
string? PublicationNonce = null)
|
||||
{
|
||||
public IReadOnlyList<string> Arguments =>
|
||||
[
|
||||
|
|
@ -59,19 +61,7 @@ public sealed class SystemBakeProcessRunner : IBakeProcessRunner
|
|||
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = request.ExecutablePath,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardInput = true,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
foreach (string argument in request.Arguments)
|
||||
{
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
ProcessStartInfo startInfo = CreateStartInfo(request);
|
||||
|
||||
using var process = new Process { StartInfo = startInfo };
|
||||
if (!process.Start())
|
||||
|
|
@ -140,6 +130,42 @@ public sealed class SystemBakeProcessRunner : IBakeProcessRunner
|
|||
}
|
||||
}
|
||||
|
||||
internal static ProcessStartInfo CreateStartInfo(BakeProcessRequest request)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
FileName = request.ExecutablePath,
|
||||
UseShellExecute = false,
|
||||
RedirectStandardInput = true,
|
||||
RedirectStandardOutput = true,
|
||||
RedirectStandardError = true,
|
||||
CreateNoWindow = true,
|
||||
};
|
||||
foreach (string argument in request.Arguments)
|
||||
{
|
||||
startInfo.ArgumentList.Add(argument);
|
||||
}
|
||||
startInfo.Environment.Remove(
|
||||
BakePublicationGuardPaths.NonceEnvironmentVariable);
|
||||
if (request.PublicationNonce is not null)
|
||||
{
|
||||
if (!BakePublicationGuardPaths.IsValidNonce(
|
||||
request.PublicationNonce))
|
||||
{
|
||||
throw new ArgumentException(
|
||||
"The bake publication nonce is invalid.",
|
||||
nameof(request));
|
||||
}
|
||||
|
||||
startInfo.Environment[
|
||||
BakePublicationGuardPaths.NonceEnvironmentVariable] =
|
||||
request.PublicationNonce;
|
||||
}
|
||||
|
||||
return startInfo;
|
||||
}
|
||||
|
||||
private static async Task PumpAsync(
|
||||
TextReader reader,
|
||||
Action<string> sink,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,120 @@
|
|||
using AcDream.Platform;
|
||||
|
||||
namespace AcDream.Launcher.Core.Installation;
|
||||
|
||||
/// <summary>
|
||||
/// Launcher half of the environment-only Bake publication guard. Paths are
|
||||
/// derived from the canonical output path, while a durable GUID nonce grants
|
||||
/// one child permission to promote its already-validated adjacent staging
|
||||
/// file. Every token mutation happens while the stable publication lock is
|
||||
/// held.
|
||||
/// </summary>
|
||||
internal static class BakePublicationGuardContract
|
||||
{
|
||||
private static readonly TimeSpan RetryDelay = TimeSpan.FromMilliseconds(50);
|
||||
|
||||
internal static async ValueTask<PublicationLease> AcquireAsync(
|
||||
string outputPath,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
string lockPath = BakePublicationGuardPaths.GetPublishLockPath(
|
||||
outputPath);
|
||||
Directory.CreateDirectory(
|
||||
Path.GetDirectoryName(lockPath)
|
||||
?? throw new InvalidOperationException(
|
||||
"The bake publication lock has no parent directory."));
|
||||
|
||||
while (true)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
try
|
||||
{
|
||||
return new PublicationLease(new FileStream(
|
||||
lockPath,
|
||||
FileMode.OpenOrCreate,
|
||||
FileAccess.ReadWrite,
|
||||
FileShare.None,
|
||||
bufferSize: 1,
|
||||
options: FileOptions.None));
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
await Task.Delay(RetryDelay, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal static void Authorize(
|
||||
string outputPath,
|
||||
string nonce,
|
||||
PublicationLease lease)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(lease);
|
||||
if (!BakePublicationGuardPaths.IsValidNonce(nonce))
|
||||
{
|
||||
throw new ArgumentException(
|
||||
"The bake publication nonce must be a lowercase GUID in N format.",
|
||||
nameof(nonce));
|
||||
}
|
||||
|
||||
string authorizationPath =
|
||||
BakePublicationGuardPaths.GetAuthorizationPath(outputPath);
|
||||
using var stream = new FileStream(
|
||||
authorizationPath,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 4096,
|
||||
options: FileOptions.WriteThrough);
|
||||
using var writer = new StreamWriter(stream, leaveOpen: true);
|
||||
writer.Write(nonce);
|
||||
writer.Flush();
|
||||
stream.Flush(flushToDisk: true);
|
||||
}
|
||||
|
||||
internal static void Invalidate(
|
||||
string outputPath,
|
||||
PublicationLease lease,
|
||||
string? onlyIfNonceMatches = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(lease);
|
||||
string authorizationPath =
|
||||
BakePublicationGuardPaths.GetAuthorizationPath(outputPath);
|
||||
if (!File.Exists(authorizationPath))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (onlyIfNonceMatches is not null)
|
||||
{
|
||||
string current = File.ReadAllText(authorizationPath);
|
||||
|
||||
if (!string.Equals(
|
||||
current,
|
||||
onlyIfNonceMatches,
|
||||
StringComparison.Ordinal))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
File.Delete(authorizationPath);
|
||||
}
|
||||
|
||||
internal sealed class PublicationLease : IAsyncDisposable
|
||||
{
|
||||
private readonly FileStream _stream;
|
||||
|
||||
internal PublicationLease(FileStream stream)
|
||||
{
|
||||
_stream = stream;
|
||||
}
|
||||
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
_stream.Dispose();
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -120,10 +120,9 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
_recordStore.DataDirectory,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
BakeOutputStagingContract.DeleteOwnedStagingFiles(
|
||||
_recordStore.PreparedAssetPath);
|
||||
InstallRecordVerification verification = await _recordStore
|
||||
.LoadAndVerifyUnderLeaseAsync(cancellationToken)
|
||||
InstallRecordVerification verification =
|
||||
await RecoverExistingUnderPublicationGuardAsync(
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
_verifiedRecord = verification.Record;
|
||||
return verification;
|
||||
|
|
@ -197,9 +196,8 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
|
||||
string outputPath = _recordStore.PreparedAssetPath;
|
||||
string backupPath = LauncherInstallRecordStore.GetBackupPath(outputPath);
|
||||
BakeOutputStagingContract.DeleteOwnedStagingFiles(outputPath);
|
||||
InstallRecordVerification existing = await _recordStore
|
||||
.LoadAndVerifyUnderLeaseAsync(cancellationToken)
|
||||
InstallRecordVerification existing =
|
||||
await RecoverExistingUnderPublicationGuardAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
_verifiedRecord = existing.Record;
|
||||
|
||||
|
|
@ -220,6 +218,7 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
|
||||
var parser = new BakeProgressJsonlParser();
|
||||
var protocol = new BakeProgressProtocol();
|
||||
string? publicationNonce = null;
|
||||
|
||||
void Observe(BakeProgressEvent progressEvent)
|
||||
{
|
||||
|
|
@ -265,11 +264,26 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
try
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
publicationNonce = BakePublicationGuardPaths.CreateNonce();
|
||||
await using (
|
||||
BakePublicationGuardContract.PublicationLease publication =
|
||||
await BakePublicationGuardContract.AcquireAsync(
|
||||
outputPath,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
BakePublicationGuardContract.Authorize(
|
||||
outputPath,
|
||||
publicationNonce,
|
||||
publication);
|
||||
}
|
||||
|
||||
var request = new BakeProcessRequest(
|
||||
_bakeExecutablePath,
|
||||
validation.Directory,
|
||||
outputPath,
|
||||
threads);
|
||||
threads,
|
||||
publicationNonce);
|
||||
BakeProcessResult processResult = await _processRunner.RunAsync(
|
||||
request,
|
||||
chunk =>
|
||||
|
|
@ -370,7 +384,11 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
.ConfigureAwait(false);
|
||||
|
||||
_verifiedRecord = record;
|
||||
LauncherInstallRecordStore.TryDelete(backupPath);
|
||||
await FinalizeSuccessfulPublicationAsync(
|
||||
outputPath,
|
||||
backupPath,
|
||||
publicationNonce)
|
||||
.ConfigureAwait(false);
|
||||
Report(
|
||||
progress,
|
||||
LauncherInstallPhase.Completed,
|
||||
|
|
@ -381,7 +399,12 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
RestorePreviousPackage(outputPath, backupPath, previousPreserved);
|
||||
await FinalizeFailedPublicationAsync(
|
||||
outputPath,
|
||||
backupPath,
|
||||
previousPreserved,
|
||||
publicationNonce)
|
||||
.ConfigureAwait(false);
|
||||
Report(
|
||||
progress,
|
||||
LauncherInstallPhase.Cancelled,
|
||||
|
|
@ -390,7 +413,12 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
RestorePreviousPackage(outputPath, backupPath, previousPreserved);
|
||||
await FinalizeFailedPublicationAsync(
|
||||
outputPath,
|
||||
backupPath,
|
||||
previousPreserved,
|
||||
publicationNonce)
|
||||
.ConfigureAwait(false);
|
||||
Report(
|
||||
progress,
|
||||
LauncherInstallPhase.Failed,
|
||||
|
|
@ -402,10 +430,62 @@ public sealed class LauncherInstaller : ILauncherInstaller
|
|||
|
||||
throw new LauncherInstallException("Installation failed.", ex);
|
||||
}
|
||||
finally
|
||||
{
|
||||
BakeOutputStagingContract.DeleteOwnedStagingFiles(outputPath);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<InstallRecordVerification>
|
||||
RecoverExistingUnderPublicationGuardAsync(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
string outputPath = _recordStore.PreparedAssetPath;
|
||||
await using BakePublicationGuardContract.PublicationLease publication =
|
||||
await BakePublicationGuardContract.AcquireAsync(
|
||||
outputPath,
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
// Any child whose parent died before it acquired this lock is now
|
||||
// irrevocably stale. A child already holding the lock must finish its
|
||||
// promotion before recovery reaches this invalidation point.
|
||||
BakePublicationGuardContract.Invalidate(outputPath, publication);
|
||||
BakeOutputStagingContract.DeleteOwnedStagingFiles(outputPath);
|
||||
return await _recordStore.LoadAndVerifyUnderLeaseAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static async Task FinalizeSuccessfulPublicationAsync(
|
||||
string outputPath,
|
||||
string backupPath,
|
||||
string publicationNonce)
|
||||
{
|
||||
await using BakePublicationGuardContract.PublicationLease publication =
|
||||
await BakePublicationGuardContract.AcquireAsync(
|
||||
outputPath,
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
BakePublicationGuardContract.Invalidate(
|
||||
outputPath,
|
||||
publication,
|
||||
publicationNonce);
|
||||
LauncherInstallRecordStore.TryDelete(backupPath);
|
||||
BakeOutputStagingContract.DeleteOwnedStagingFiles(outputPath);
|
||||
}
|
||||
|
||||
private static async Task FinalizeFailedPublicationAsync(
|
||||
string outputPath,
|
||||
string backupPath,
|
||||
bool previousPreserved,
|
||||
string? publicationNonce)
|
||||
{
|
||||
await using BakePublicationGuardContract.PublicationLease publication =
|
||||
await BakePublicationGuardContract.AcquireAsync(
|
||||
outputPath,
|
||||
CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
BakePublicationGuardContract.Invalidate(
|
||||
outputPath,
|
||||
publication,
|
||||
publicationNonce);
|
||||
RestorePreviousPackage(outputPath, backupPath, previousPreserved);
|
||||
BakeOutputStagingContract.DeleteOwnedStagingFiles(outputPath);
|
||||
}
|
||||
|
||||
private bool PreservePreviousPackage(string outputPath, string backupPath)
|
||||
|
|
|
|||
35
src/AcDream.Platform/BakePublicationGuardPaths.cs
Normal file
35
src/AcDream.Platform/BakePublicationGuardPaths.cs
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
namespace AcDream.Platform;
|
||||
|
||||
/// <summary>
|
||||
/// Portable, versioned naming contract shared by the launcher parent and the
|
||||
/// independently published bake child. The durable token grants one child
|
||||
/// permission to publish while the adjacent OS-held lock serializes its final
|
||||
/// promotion with launcher recovery.
|
||||
/// </summary>
|
||||
public static class BakePublicationGuardPaths
|
||||
{
|
||||
public const string NonceEnvironmentVariable =
|
||||
"ACDREAM_BAKE_PUBLISH_NONCE_V1";
|
||||
public const string PublishLockSuffix = ".publish.lock";
|
||||
public const string AuthorizationSuffix = ".publish-token";
|
||||
|
||||
public static string CreateNonce() => Guid.NewGuid().ToString("N");
|
||||
|
||||
public static bool IsValidNonce(string? nonce) =>
|
||||
nonce is not null
|
||||
&& nonce.Length == 32
|
||||
&& Guid.TryParseExact(nonce, "N", out Guid parsed)
|
||||
&& string.Equals(parsed.ToString("N"), nonce, StringComparison.Ordinal);
|
||||
|
||||
public static string GetPublishLockPath(string outputPath)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(outputPath);
|
||||
return Path.GetFullPath(outputPath) + PublishLockSuffix;
|
||||
}
|
||||
|
||||
public static string GetAuthorizationPath(string outputPath)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(outputPath);
|
||||
return Path.GetFullPath(outputPath) + AuthorizationSuffix;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue