nucleic-windows/spikes/WslcSpike/Program.cs

using System.Diagnostics;
using System.Text.Json;

namespace WslcSpike;

/// <summary>
/// M1 spike (a2) — the wslc happy path, end to end, through the real broker
/// (docs/WINDOWS_PORT.md §13).
///
/// Everything in the Windows container subsystem now compiles and none of it has ever run:
/// `WslcFacade` (the compat SDK arm), `WslcInternal` (D13 Tier 1 recovery), the §3.3 RPC surface.
/// This drives all of it as hostd would — spawn `nucleic-brokerd.exe`, speak NDJSON over stdio —
/// so what passes here is the shipping code, not a rehearsal of it.
///
/// It answers three questions the plan is still guessing at:
///
/// 1. **Does the happy path work at all?** session → GHCR pull → container with an NTFS
///    `ContainerVolume` → exec → stdio round-trip → signal → teardown.
/// 2. **Is 9P fast enough to put repos on NTFS (D8)?** §15 lists this as a top risk with no
///    numbers behind it. The spike times `git status` in the bind-mounted worktree AND in a copy
///    on the container's own ext4, which is the comparison that actually isolates the mount.
/// 3. **Does D13 Tier 1 recovery work?** Kill the broker with the session up, start another, and
///    see whether it clears the orphan instead of failing `session_exists`.
///
/// Nothing here is asserted-and-exits: every step prints what happened, because a spike's product
/// is evidence. It exits non-zero only if a step that should work threw.
/// </summary>
internal static class Program
{
    private static async Task<int> Main(string[] args)
    {
        var repo = Arg(args, "--repo") ?? Directory.GetCurrentDirectory();
        var image = Arg(args, "--image") ?? "ghcr.io/abkslm/naros-agent:26.07";
        var container = Arg(args, "--container") ?? "nucleic-spike-c1";
        var sessionName = Arg(args, "--session-name") ?? "nucleic-spike";
        var iterations = int.TryParse(Arg(args, "--iterations"), out var n) ? n : 5;
        var broker = Arg(args, "--broker") ?? FindBroker();
        var skipRecovery = args.Contains("--no-recovery");
        // narOS runs `naros-init` as PID 1 and stays up (docs/NAROS.md). A stock image usually
        // does not: alpine's PID 1 is /bin/sh, which exits immediately with no tty, so the
        // container is `Exited` before the first exec and every later step fails `not_running`.
        var sleepInit = args.Contains("--sleep-init");
        var verbose = args.Contains("--verbose");

        if (broker is null || !File.Exists(broker))
        {
            Console.Error.WriteLine(
                "could not find nucleic-brokerd.exe. Build it first:\n"
                + "  dotnet build windows\\NucleicBroker\\NucleicBroker.csproj -p:UseWslc=true\n"
                + "then pass --broker <path> if it still isn't found.");
            return 2;
        }

        Console.WriteLine($"broker : {broker}");
        Console.WriteLine($"repo   : {repo}");
        Console.WriteLine($"image  : {image}");
        Console.WriteLine();

        try
        {
            await RunScenarioAsync(broker, sessionName, image, container, repo, iterations, sleepInit, verbose);
            if (!skipRecovery) await RunRecoveryAsync(broker, sessionName);
            return 0;
        }
        catch (Exception e)
        {
            Console.Error.WriteLine();
            Console.Error.WriteLine($"FAILED: {e.Message}");
            Console.Error.WriteLine(
                "The [brokerd] lines above are the facade's own diagnostics and usually say why.");
            return 1;
        }
    }

    private static async Task RunScenarioAsync(
        string brokerPath, string sessionName, string image, string containerName,
        string repo, int iterations, bool sleepInit, bool verbose)
    {
        await using var broker = BrokerClient.Spawn(brokerPath);
        broker.Verbose = verbose;

        // Pull progress is high-rate; collapse it to one line per phase change so the transcript
        // stays readable but a stalled pull is still visible.
        var lastStatus = "";
        broker.Notification += (method, args) =>
        {
            switch (method)
            {
                case "image.pullProgress":
                    var status = args.TryGetProperty("status", out var s) ? s.GetString() ?? "" : "";
                    if (status != lastStatus) { lastStatus = status; Console.WriteLine($"    pull: {status}"); }
                    break;
                case "session.down":
                    Console.WriteLine($"    !! session.down: {args}");
                    break;
            }
        };

        Step("hello");
        var hello = await broker.CallAsync("hello");
        var capabilities = hello.GetProperty("capabilities").EnumerateArray()
            .Select(c => c.GetString()).ToList();
        Console.WriteLine($"    brokerd {hello.GetProperty("brokerVersion").GetString()}, "
            + $"wslc {hello.GetProperty("wslcVersion").GetString() ?? "(absent)"}");
        Console.WriteLine($"    capabilities: {string.Join(", ", capabilities)}");
        // These are the D13 capability flags; say plainly which arm answered so the run is
        // self-describing rather than needing the doc open beside it.
        Console.WriteLine(capabilities.Contains("recover")
            ? "    → D13 Tier 1 bound: an orphaned session will be recovered automatically"
            : "    → D13 Tier 1 did NOT bind: a broker restart will need `wsl --shutdown`");

        Step("components.missing");
        var missing = (await broker.CallAsync("components.missing")).GetProperty("flags")
            .EnumerateArray().Select(c => c.GetString()).ToList();
        if (missing.Count > 0)
            throw new InvalidOperationException(
                $"this machine cannot run wslc yet — missing {string.Join(", ", missing)}. "
                + "See windows/spikes/README.md for the per-component remedy.");
        Console.WriteLine("    none missing");

        Step("session.ensure");
        var dataDir = Path.Combine(
            Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
            "Nucleic", "spike", "wslc");
        Directory.CreateDirectory(dataDir);
        var gateway = (await broker.CallAsync("session.ensure",
            new { name = sessionName, dataDir, cpu = 4, memoryMB = 8192 }, 300))
            .GetProperty("gateway").GetString();
        // §5's control plane depends on this address entirely, and no wslc API surfaces it —
        // it comes from the WSL vEthernet adapter. If it is empty or a loopback, the control
        // plane cannot work and M1 (b) has its answer early.
        Console.WriteLine($"    host gateway (guest→host, §5): {gateway}");

        Step($"image.pull {image}");
        var pullWatch = Stopwatch.StartNew();
        await broker.CallAsync("image.pull", new { @ref = image }, 1800);
        Console.WriteLine($"    pulled in {pullWatch.Elapsed.TotalSeconds:F1}s");

        Step($"container.create {containerName}");
        var create = new Dictionary<string, object?>
        {
            ["name"] = containerName,
            ["image"] = image,
            ["volumes"] = new[] { new { host = repo, guest = "/work", ro = false } },
            ["networkingMode"] = "Bridged",
            ["hostname"] = "nucleic-spike",
            ["env"] = new Dictionary<string, string> { ["NUCLEIC_SPIKE"] = "1" },
        };
        if (sleepInit) create["initArgv"] = new[] { "/bin/sh", "-c", "sleep 3600" };
        await broker.CallAsync("container.create", create);
        await broker.CallAsync("container.start", new { name = containerName });
        var state = (await broker.CallAsync("container.state", new { name = containerName }))
            .GetProperty("state").GetString();
        Console.WriteLine($"    state: {state}");
        // Check here rather than letting the first exec fail `not_running`: a container whose
        // PID 1 exited looks identical to a container that failed to start, and the remedy
        // (supply a long-running init) is nothing like the remedy for a real start failure.
        if (state != "running")
            throw new InvalidOperationException(
                $"container is '{state}' immediately after start — its PID 1 exited. narOS runs "
                + "naros-init and stays up; a stock image (alpine's PID 1 is /bin/sh) does not. "
                + "Re-run with --sleep-init to give it a long-running init process.");

        Step("exec: stdio round-trip");
        var (code, output) = await ExecAsync(broker, containerName, ["/bin/sh", "-c", "echo hello from \"$(uname -n)\"; echo to-stderr 1>&2"]);
        Console.WriteLine($"    exit {code}: {output.Trim()}");

        Step("exec: uid drop (setpriv wrapper, §3.2)");
        // ProcessSettings has no uid/gid, so the facade wraps argv in setpriv. A BusyBox image has
        // a setpriv that cannot change uid, and the facade now REFUSES rather than silently
        // running the agent as root — so `unsupported` here is correct behaviour on such an image,
        // not a failure of the run.
        try
        {
            var (idCode, idOut) = await ExecAsync(
                broker, containerName, ["/bin/sh", "-c", "id -u; id -g"], uid: 501, gid: 501);
            Console.WriteLine(idCode == 0
                ? $"    uid/gid inside container: {idOut.Replace("\n", "/").Trim('/')}"
                : $"    ran but reported failure (exit {idCode}): {idOut.Trim()}");
        }
        catch (BrokerError e) when (e.Kind == "unsupported")
        {
            Console.WriteLine($"    REFUSED (correctly): {e.Message}");
            Console.WriteLine("    → expected on a BusyBox image; narOS ships util-linux.");
        }

        Step("the mounted worktree");
        var (lsCode, lsOut) = await ExecAsync(broker, containerName,
            ["/bin/sh", "-c", "ls /work | head -5; echo ---; test -d /work/.git && echo 'git repo present'"]);
        Console.WriteLine($"    exit {lsCode}\n{Indent(lsOut)}");

        await MeasureNinePAsync(broker, containerName, iterations);

        Step("container.stats (cgroup read — no GetStatistics() exists)");
        var stats = (await broker.CallAsync("container.stats", new { name = containerName }))
            .GetProperty("stats");
        Console.WriteLine(stats.ValueKind == JsonValueKind.Null
            ? "    null (container not running?)"
            : $"    {stats}");

        Step("teardown");
        await broker.CallAsync("container.stop", new { name = containerName, signal = 15, graceMs = 10000 });
        await broker.CallAsync("container.delete", new { name = containerName, force = true });
        Console.WriteLine("    stopped and deleted");
    }

    /// <summary>
    /// §15's top unquantified risk: D8 puts repos on NTFS and bind-mounts them, so every git and
    /// npm operation crosses 9P. The only honest measurement is the same work on both sides of
    /// the mount, in the same container, on the same repo — so this copies the worktree to the
    /// container's own ext4 and runs the identical commands there.
    /// </summary>
    private static async Task MeasureNinePAsync(BrokerClient broker, string container, int iterations)
    {
        // `find -type f` rather than `git status`, because it needs only busybox and measures the
        // same thing that makes git slow over a mount: a full lstat() traversal of the tree. Using
        // git would make the number depend on the image having git, which is what derailed the
        // first attempt at this measurement.
        var probe = "find . -type f | wc -l";
        var (gitCode, _) = await ExecAsync(broker, container, ["/bin/sh", "-c", "command -v git"]);
        if (gitCode == 0)
        {
            probe = "git status --porcelain >/dev/null";
            Console.WriteLine("    (git present — measuring `git status`)");
        }
        else
        {
            Console.WriteLine("    (no git in this image — measuring `find -type f`, which is the "
                + "lstat traversal that dominates `git status`)");
        }

        Step($"9P vs ext4 — `{probe}` x{iterations} (§15 risk, D8)");

        var mounted = await TimeCommandAsync(broker, container, $"cd /work && {probe}", iterations);
        Report("/work (NTFS via 9P)", mounted);

        Console.WriteLine("    copying the worktree to container-local ext4…");
        var copyWatch = Stopwatch.StartNew();
        var (copyCode, copyOut) = await ExecAsync(broker, container,
            ["/bin/sh", "-c", "rm -rf /tmp/ext4 && cp -a /work /tmp/ext4 && echo ok"], timeoutSeconds: 1800);
        if (copyCode != 0)
        {
            Console.WriteLine($"    copy failed (exit {copyCode}): {copyOut.Trim()} — skipping the ext4 leg");
            return;
        }
        Console.WriteLine($"    copied in {copyWatch.Elapsed.TotalSeconds:F1}s");

        var local = await TimeCommandAsync(broker, container, $"cd /tmp/ext4 && {probe}", iterations);
        Report("/tmp/ext4 (container-local)", local);

        if (mounted.Count > 0 && local.Count > 0)
        {
            var ratio = Median(mounted) / Math.Max(1, Median(local));
            Console.WriteLine($"    → 9P is {ratio:F1}x the local time (median)");
            // The threshold is a judgement call, not a measurement, so it is stated as one.
            Console.WriteLine(ratio switch
            {
                < 2 => "    → acceptable: D8 stands, repos stay on NTFS.",
                < 5 => "    → noticeable. D8 stands, but cache dirs (node_modules, build output)\n"
                       + "      should go on a ContainerNamedVolume as §15 anticipated.",
                _ => "    → BAD. §15 says surface this to the user for a decision rather than\n"
                     + "      silently moving repos into ext4. Re-measure on a large repo first.",
            });
        }

        await ExecAsync(broker, container, ["/bin/sh", "-c", "rm -rf /tmp/ext4"]);
    }

    private static async Task<List<double>> TimeCommandAsync(
        BrokerClient broker, string container, string script, int iterations)
    {
        var timings = new List<double>();
        for (var i = 0; i < iterations; i++)
        {
            var watch = Stopwatch.StartNew();
            var (code, output) = await ExecAsync(broker, container, ["/bin/sh", "-c", script], timeoutSeconds: 600);
            watch.Stop();
            if (code != 0)
            {
                Console.WriteLine($"    command failed (exit {code}): {output.Trim()}");
                return timings;
            }
            timings.Add(watch.Elapsed.TotalMilliseconds);
        }
        return timings;
    }

    private static void Report(string label, List<double> timings)
    {
        if (timings.Count == 0) { Console.WriteLine($"    {label}: no successful runs"); return; }
        Console.WriteLine($"    {label}: median {Median(timings):F0}ms  "
            + $"(min {timings.Min():F0}, max {timings.Max():F0}, n={timings.Count})");
    }

    private static double Median(List<double> values)
    {
        var sorted = values.Order().ToList();
        return sorted.Count % 2 == 1
            ? sorted[sorted.Count / 2]
            : (sorted[sorted.Count / 2 - 1] + sorted[sorted.Count / 2]) / 2;
    }

    /// <summary>
    /// D13 Tier 1, live (docs/WINDOWS_PORT.md §13.2). Kill a broker with its session up — the
    /// exact §2.3 crash — then start a fresh one and ensure the same session name. Before Tier 1
    /// this failed `session_exists` and needed a manual `wsl --shutdown`; it should now recover.
    /// </summary>
    private static async Task RunRecoveryAsync(string brokerPath, string sessionName)
    {
        Step("D13 Tier 1: kill the broker, then recover the orphaned session");

        var dataDir = Path.Combine(
            Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
            "Nucleic", "spike", "wslc");

        await using (var first = BrokerClient.Spawn(brokerPath))
        {
            await first.CallAsync("hello");
            await first.CallAsync("session.ensure", new { name = sessionName, dataDir }, 300);
            Console.WriteLine("    session up; killing brokerd without shutdown…");
            first.Kill();
        }
        // The session is deliberately left running: that is the orphan.
        await Task.Delay(2000);

        await using var second = BrokerClient.Spawn(brokerPath);
        var hello = await second.CallAsync("hello");
        var canRecover = hello.GetProperty("capabilities").EnumerateArray()
            .Any(c => c.GetString() == "recover");
        try
        {
            var gateway = (await second.CallAsync(
                "session.ensure", new { name = sessionName, dataDir }, 300))
                .GetProperty("gateway").GetString();
            Console.WriteLine($"    RECOVERED — fresh session up, gateway {gateway}");
            Console.WriteLine("    → a broker crash no longer strands the sandbox.");
            await second.CallAsync("session.terminate");
        }
        catch (BrokerError e) when (e.Kind == "session_exists")
        {
            Console.WriteLine($"    NOT recovered: {e.Message}");
            Console.WriteLine(canRecover
                ? "    → Tier 1 bound but did not clear the orphan. This is the bug to chase:\n"
                  + "      read the [brokerd] lines above for the OpenSessionByName/Terminate result."
                : "    → expected: Tier 1 never bound on this machine (see the hello above).");
            Console.WriteLine("    Clear it manually with `wsl --shutdown`.");
        }
    }

    // MARK: - Helpers

    /// <summary>Run argv to completion, collecting stdout+stderr from the broker's notifications
    /// exactly as `WslcProcessHandle` does.</summary>
    private static async Task<(int Code, string Output)> ExecAsync(
        BrokerClient broker, string container, string[] argv,
        int? uid = null, int? gid = null, int timeoutSeconds = 300)
    {
        var output = new MemoryStream();
        var exited = new TaskCompletionSource<int>(TaskCreationOptions.RunContinuationsAsynchronously);

        // Deliberately NOT filtered by procId.
        //
        // The broker now guarantees the `proc.exec` response precedes any notification, but that
        // is not sufficient for a client: completing the response's TaskCompletionSource only
        // SCHEDULES the awaiting continuation, so the reader thread can dispatch the very next
        // line — proc.stdout, or proc.exit — before the continuation has recorded the procId.
        // Filtering on a not-yet-assigned procId silently drops the exit and hangs forever, which
        // is exactly how this spike hung twice (docs/WINDOWS_PORT.md §13.3).
        //
        // Safe here because the spike runs one process at a time and awaits each to completion.
        // A real client cannot take this shortcut: it must buffer notifications for procIds it
        // has not yet learned. Worth checking `WslcBrokerClient.swift` for the same race.
        void OnNotification(string method, JsonElement args)
        {
            if (!args.TryGetProperty("procId", out _)) return;
            switch (method)
            {
                case "proc.stdout":
                case "proc.stderr":
                    var chunk = Convert.FromBase64String(args.GetProperty("b64").GetString()!);
                    lock (output) output.Write(chunk, 0, chunk.Length);
                    break;
                case "proc.exit":
                    exited.TrySetResult(args.GetProperty("code").GetInt32());
                    break;
            }
        }

        broker.Notification += OnNotification;
        try
        {
            object spec = uid is null
                ? new { container, argv, tty = false }
                : new { container, argv, uid, gid = gid ?? uid, tty = false };
            await broker.CallAsync("proc.exec", spec);

            var code = await exited.Task.WaitAsync(TimeSpan.FromSeconds(timeoutSeconds));
            lock (output) return (code, System.Text.Encoding.UTF8.GetString(output.ToArray()));
        }
        finally
        {
            broker.Notification -= OnNotification;
        }
    }

    /// <summary>Locate the broker built with -p:UseWslc=true. The TFM is windows-specific there,
    /// so glob rather than hardcoding a path that moves with the SDK pin.</summary>
    private static string? FindBroker()
    {
        var here = new DirectoryInfo(AppContext.BaseDirectory);
        for (var dir = here; dir is not null; dir = dir.Parent)
        {
            var candidate = Path.Combine(dir.FullName, "windows", "NucleicBroker", "bin");
            if (!Directory.Exists(candidate)) continue;
            return Directory
                .EnumerateFiles(candidate, "nucleic-brokerd.exe", SearchOption.AllDirectories)
                // Prefer the windows-TFM build: that is the one with USE_WSLC compiled in.
                .OrderByDescending(p => p.Contains("windows10.0"))
                .ThenByDescending(File.GetLastWriteTimeUtc)
                .FirstOrDefault();
        }
        return null;
    }

    private static void Step(string title)
    {
        Console.WriteLine();
        Console.WriteLine($"==> {title}");
    }

    private static string Indent(string text) =>
        string.Join("\n", text.Split('\n').Select(l => "      " + l));

    private static string? Arg(string[] args, string flag)
    {
        var i = Array.IndexOf(args, flag);
        return i >= 0 && i + 1 < args.Length && !args[i + 1].StartsWith("--") ? args[i + 1] : null;
    }
}