acdream/tools/ShaderCompiler/Program.cs

using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using Silk.NET.Core.Native;
using Silk.NET.Shaderc;

namespace AcDream.Tools.ShaderCompiler;

/// <summary>
/// Campaign V slice V6c, plan §4.6: compile acdream's GLSL to committed SPIR-V.
///
/// <para>Usage: <c>ShaderCompiler &lt;shaders-dir&gt; &lt;output-dir&gt;</c>.
/// Every <c>.vert</c>/<c>.frag</c> pair in the source directory is compiled
/// through <see cref="VulkanGlslPreamble"/>; successes write
/// <c>{name}.{stage}.spv</c> and failures are recorded with the compiler's own
/// message. Either way the run writes <c>shaders.manifest.json</c> containing
/// the SHA-256 of every GLSL input, which the App test suite re-checks so a
/// source edit that never got recompiled fails a test rather than shipping a
/// stale binary.</para>
///
/// <para>The exit code is 0 when every shader the manifest expects to succeed
/// did. A shader that is not yet Vulkan-expressible is not a build failure — it
/// is a fact about which renderer-port slices are still outstanding, recorded in
/// the manifest so it is reviewable rather than folklore.</para>
/// </summary>
internal static class Program
{
    private static unsafe int Main(string[] args)
    {
        if (args.Length < 2
            || args.Length > 3
            || (args.Length == 3 && !string.Equals(args[2], "--force", StringComparison.Ordinal)))
        {
            Console.Error.WriteLine("usage: ShaderCompiler <shaders-dir> <output-dir> [--force]");
            return 2;
        }

        string sourceDirectory = Path.GetFullPath(args[0]);
        string outputDirectory = Path.GetFullPath(args[1]);
        bool force = args.Length == 3;
        Directory.CreateDirectory(outputDirectory);
        string manifestPath = Path.Combine(outputDirectory, "shaders.manifest.json");
        IReadOnlyDictionary<(string Name, string Stage), ShaderStageResult> previousStages =
            force ? new Dictionary<(string, string), ShaderStageResult>() : LoadPreviousStages(manifestPath);

        string[] names = Directory
            .EnumerateFiles(sourceDirectory, "*.vert")
            .Select(Path.GetFileNameWithoutExtension)
            .Where(name => name is not null)
            .Select(name => name!)
            .Where(name => File.Exists(Path.Combine(sourceDirectory, $"{name}.frag")))
            .OrderBy(name => name, StringComparer.Ordinal)
            .ToArray();

        var shaderc = Shaderc.GetApi();
        Compiler* compiler = shaderc.CompilerInitialize();
        CompileOptions* options = shaderc.CompileOptionsInitialize();
        shaderc.CompileOptionsSetTargetEnv(options, TargetEnv.Vulkan, (uint)EnvVersion.Vulkan13);
        shaderc.CompileOptionsSetTargetSpirv(options, SpirvVersion.Shaderc16);
        // Performance rather than size: these are compiled once, committed, and
        // then loaded by every launch forever.
        shaderc.CompileOptionsSetOptimizationLevel(options, OptimizationLevel.Performance);

        var entries = new List<ShaderManifestEntry>();
        int failures = 0;
        try
        {
            foreach (string name in names)
            {
                var stages = new List<ShaderStageResult>();
                // Shared across the pair so a fragment input takes the location
                // its vertex output was given, by name.
                var locations = new Dictionary<string, int>(StringComparer.Ordinal);
                foreach (string stage in (string[])["vert", "frag"])
                {
                    string path = Path.Combine(sourceDirectory, $"{name}.{stage}");
                    string source = File.ReadAllText(path);
                    if (source.Contains("#include \"", StringComparison.Ordinal))
                        source = GlslIncludeExpander.Expand(source, sourceDirectory);
                    string hash = Sha256(source);
                    string target = Path.Combine(outputDirectory, $"{name}.{stage}.spv");

                    // An unchanged source already has the exact committed artifact
                    // described by the prior manifest. Keeping it avoids compiler-
                    // version-only SPIR-V decoration reordering on the retail path;
                    // a real source edit changes the hash and recompiles normally.
                    if (previousStages.TryGetValue((name, stage), out ShaderStageResult? previous)
                        && previous.Compiled
                        && string.Equals(previous.SourceSha256, hash, StringComparison.Ordinal)
                        && File.Exists(target))
                    {
                        stages.Add(new ShaderStageResult(stage, hash, true, null));
                        continue;
                    }

                    string transformed;
                    try
                    {
                        transformed = GlslVaryingLocations.Apply(
                            VulkanGlslPreamble.Apply(source, stage),
                            stage,
                            locations);
                    }
                    catch (Exception error)
                    {
                        stages.Add(new ShaderStageResult(stage, hash, false, error.Message));
                        continue;
                    }

                    if (TryCompile(shaderc, compiler, options, transformed, $"{name}.{stage}", stage, out byte[] spirv, out string message))
                    {
                        File.WriteAllBytes(target, spirv);
                        stages.Add(new ShaderStageResult(stage, hash, true, null));
                    }
                    else
                    {
                        if (File.Exists(target))
                            File.Delete(target);
                        stages.Add(new ShaderStageResult(stage, hash, false, Summarise(message)));
                    }
                }

                bool ok = stages.All(stage => stage.Compiled);
                if (!ok)
                {
                    failures++;
                    // Half a pair is worse than none: the device would load the
                    // surviving stage happily and nobody could account for it.
                    foreach (string stage in (string[])["vert", "frag"])
                    {
                        string orphan = Path.Combine(outputDirectory, $"{name}.{stage}.spv");
                        if (File.Exists(orphan))
                            File.Delete(orphan);
                    }
                }
                entries.Add(new ShaderManifestEntry(name, ok, stages));
                Console.WriteLine(ok
                    ? $"[shaders] {name}: ok"
                    : $"[shaders] {name}: NOT VULKAN-EXPRESSIBLE YET — {FirstReason(stages)}");
            }
        }
        finally
        {
            // shaderc's own handles are released; the Silk.NET API container
            // deliberately is not. Disposing the container unloads the native
            // module, and on Linux dlclose'ing libshaderc_shared.so kills the
            // process during exit: glslang registers process-level teardown
            // that runs after the module's code has been unmapped. Bisected on
            // Ubuntu 24.04 with a four-mode probe - GetApi, CompilerInitialize
            // and CompilerRelease all exit 0, and adding only the container
            // Dispose turns the exit into SIGSEGV (139), which reached CI as
            // "shader compilation failed with exit code 134". Nothing is leaked
            // by omitting it: the module's lifetime is the process's, and the
            // process is one statement from returning.
            shaderc.CompileOptionsRelease(options);
            shaderc.CompilerRelease(compiler);
        }

        var manifest = new ShaderManifest(
            "Campaign V slice V6c. Regenerate with tools/compile-shaders.ps1.",
            entries.OrderBy(entry => entry.Name, StringComparer.Ordinal).ToList());
        File.WriteAllText(
            manifestPath,
            JsonSerializer.Serialize(manifest, ShaderManifestJson.Options) + Environment.NewLine);

        Console.WriteLine(
            $"[shaders] {entries.Count - failures}/{entries.Count} pair(s) compiled; manifest at {manifestPath}");
        return 0;
    }

    private static unsafe bool TryCompile(
        Shaderc shaderc,
        Compiler* compiler,
        CompileOptions* options,
        string source,
        string name,
        string stage,
        out byte[] spirv,
        out string message)
    {
        ShaderKind kind = stage == "vert" ? ShaderKind.VertexShader : ShaderKind.FragmentShader;
        byte[] sourceBytes = Encoding.UTF8.GetBytes(source);
        byte[] nameBytes = Encoding.UTF8.GetBytes(name + "\0");
        byte[] entryBytes = Encoding.UTF8.GetBytes("main\0");

        fixed (byte* sourcePointer = sourceBytes)
        fixed (byte* namePointer = nameBytes)
        fixed (byte* entryPointer = entryBytes)
        {
            CompilationResult* result = shaderc.CompileIntoSpv(
                compiler,
                sourcePointer,
                (nuint)sourceBytes.Length,
                kind,
                namePointer,
                entryPointer,
                options);
            try
            {
                CompilationStatus status = shaderc.ResultGetCompilationStatus(result);
                message = SilkMarshal.PtrToString((nint)shaderc.ResultGetErrorMessage(result)) ?? string.Empty;
                if (status != CompilationStatus.Success)
                {
                    spirv = [];
                    return false;
                }

                nuint length = shaderc.ResultGetLength(result);
                spirv = new byte[(int)length];
                new ReadOnlySpan<byte>(shaderc.ResultGetBytes(result), (int)length).CopyTo(spirv);
                return true;
            }
            finally
            {
                shaderc.ResultRelease(result);
            }
        }
    }

    /// <summary>First line of a compiler message, which is the one that names the cause.</summary>
    private static string Summarise(string message)
    {
        string[] lines = message
            .Replace("\r\n", "\n")
            .Split('\n', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
        return lines.Length == 0 ? "unknown compiler failure" : lines[0];
    }

    private static string FirstReason(IEnumerable<ShaderStageResult> stages) =>
        stages.FirstOrDefault(stage => !stage.Compiled)?.Message ?? "unknown";

    private static string Sha256(string text)
    {
        // Line endings are normalised before hashing so a checkout with
        // different core.autocrlf settings does not report every shader stale.
        byte[] bytes = Encoding.UTF8.GetBytes(text.Replace("\r\n", "\n"));
        return Convert.ToHexStringLower(SHA256.HashData(bytes));
    }

    private static IReadOnlyDictionary<(string Name, string Stage), ShaderStageResult> LoadPreviousStages(
        string manifestPath)
    {
        if (!File.Exists(manifestPath))
            return new Dictionary<(string, string), ShaderStageResult>();

        try
        {
            ShaderManifest? manifest = JsonSerializer.Deserialize<ShaderManifest>(
                File.ReadAllText(manifestPath),
                ShaderManifestJson.Options);
            return manifest?.Shaders
                .SelectMany(shader => shader.Stages.Select(stage => (shader.Name, Stage: stage)))
                .ToDictionary(item => (item.Name, item.Stage.Stage), item => item.Stage)
                ?? new Dictionary<(string, string), ShaderStageResult>();
        }
        catch (JsonException)
        {
            // A malformed or obsolete manifest is not an authority. Recompile
            // everything and replace it with the current deterministic schema.
            return new Dictionary<(string, string), ShaderStageResult>();
        }
    }
}

internal sealed record ShaderStageResult(
    [property: JsonPropertyName("stage")] string Stage,
    [property: JsonPropertyName("sourceSha256")] string SourceSha256,
    [property: JsonPropertyName("compiled")] bool Compiled,
    [property: JsonPropertyName("message")] string? Message);

internal sealed record ShaderManifestEntry(
    [property: JsonPropertyName("name")] string Name,
    [property: JsonPropertyName("vulkanReady")] bool VulkanReady,
    [property: JsonPropertyName("stages")] IReadOnlyList<ShaderStageResult> Stages);

internal sealed record ShaderManifest(
    [property: JsonPropertyName("note")] string Note,
    [property: JsonPropertyName("shaders")] IReadOnlyList<ShaderManifestEntry> Shaders);

internal static class ShaderManifestJson
{
    internal static JsonSerializerOptions Options { get; } = new()
    {
        WriteIndented = true,
        DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
    };
}