#!/usr/bin/env python3 """Differential replay harness for the macOS HOST surface (docs/NASH.md §10.6 — the M5.5 gate). The host risk is *semantic divergence*, not parse failure: a command that parses under nash but means something different under zsh (unquoted word splitting, no-match globbing) runs to completion with different results and no fallback event. So this harness replays each corpus command under BOTH `zsh -lc` (today's host shell) and `nash -c` (the M5.5 flip: login sourcing is replaced by the LoginShellEnv snapshot, hence `-c`) in identical disposable workspaces, then diffs exit code, stdout, AND the resulting file-system tree. stderr is compared separately and never counts against parity (error wording legitimately differs). Corpus entries (host-corpus.jsonl, one JSON object per line): {"id": "...", "cmd": "...", # a replayable command "class": "replayable" | "excluded:", # exclusions are REPORTED, never silent "expect": "match" | "diverge"} # 'diverge' = harness SELF-TEST entry: a known # zsh≠bash-family semantic difference the # diff engine MUST catch (else the engine is # blind and the gate is meaningless) Workspaces: a synthetic fixture tree by default; pass --worktree PATH to clone a real project tree instead (cp -R into the scratch dir) so toolchain commands (`swift build`, `make`) replay against real inputs — §10.6's "disposable clone". Verifying the harness before a darwin nash exists: `--nash /bin/bash` is a faithful stand-in for the divergence semantics under test (bash word-splits and passes unmatched globs through exactly as nash's brush core does; zsh does neither), so the self-test entries must report DIVERGE and real entries' verdicts are meaningful. The real gate run uses the built nash. Exit status: 0 iff every replayable expect-match entry matched AND every expect-diverge self-test diverged AND at least one self-test ran. """ import argparse import hashlib import json import os import shutil import subprocess import sys import tempfile FIXTURES = { "a.txt": "first line\nsecond line\nthird line\n", "b.txt": "banana\napple\nbanana\ncherry\napple\n", "data.csv": "name,team,score\nalice,red,10\nbob,blue,7\ncarol,red,9\ndave,blue,7\n", "logs/app.log": ( "2026-01-01 INFO boot ok\n" "2026-01-01 ERROR disk full\n" "2026-01-01 INFO retry\n" "2026-01-01 WARN slow\n" "2026-01-01 ERROR net down\n" "2026-01-01 INFO done\n" ), "src/main.py": "print('main')\n", "src/util.py": "def add(a, b):\n return a + b\n", "README.md": "# Sample\n", "Makefile": "check:\n\t@echo make-ran\n", } TIMEOUT_S = 60 def seed(workdir, worktree=None): if worktree: shutil.copytree(worktree, workdir, symlinks=True, dirs_exist_ok=True) return for rel, content in FIXTURES.items(): path = os.path.join(workdir, rel) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w") as f: f.write(content) def snapshot(workdir): """Relative path -> sha256 of contents, for every file in the tree.""" out = {} for root, _dirs, files in os.walk(workdir): for name in files: path = os.path.join(root, name) rel = os.path.relpath(path, workdir) try: with open(path, "rb") as f: out[rel] = hashlib.sha256(f.read()).hexdigest() except OSError: out[rel] = "" return out def run_one(shell, mode, cmd, worktree=None): parent = tempfile.mkdtemp(prefix="nash-hostdiff-") workdir = os.path.join(parent, "workspace") os.makedirs(workdir, exist_ok=True) seed(workdir, worktree) # A pinned, minimal env on purpose: the point of §7.7.2 is that the flip must NOT depend on # what login sourcing would add — a command that only works under `zsh -lc`'s rc-sourced env # shows up here as a divergence to investigate, which is the gate doing its job. env = { "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "HOME": workdir, "LC_ALL": "C", "LANG": "C", "TERM": "dumb", "SHELL": shell, } try: proc = subprocess.run( [shell, mode, cmd], cwd=workdir, env=env, capture_output=True, timeout=TIMEOUT_S ) code, out, err = proc.returncode, proc.stdout, proc.stderr except subprocess.TimeoutExpired: code, out, err = "TIMEOUT", b"", b"" fs = snapshot(workdir) shutil.rmtree(parent, ignore_errors=True) def norm(b): text = b.decode("utf-8", "replace") return text.replace(workdir, "__WORK__").replace(parent, "__TMP__") return {"code": code, "stdout": norm(out), "stderr": norm(err), "fs": fs} def classify_outcome(z, n): core_equal = ( z["code"] == n["code"] and z["stdout"] == n["stdout"] and z["fs"] == n["fs"] ) if core_equal and z["stderr"] == n["stderr"]: return "MATCH" if core_equal: return "STDERR_ONLY" return "DIVERGE" def main(): ap = argparse.ArgumentParser() ap.add_argument("--nash", default=os.environ.get("NASH_BIN", "nash"), help="nash binary (or /bin/bash as a semantics stand-in pre-nash)") ap.add_argument("--zsh", default="/bin/zsh") ap.add_argument( "--corpus", default=os.path.join( os.path.dirname(os.path.abspath(__file__)), "host-corpus.jsonl"), ) ap.add_argument("--worktree", help="clone this tree as the workspace (disposable copy)") ap.add_argument("--json", help="also write the full report to this path") args = ap.parse_args() entries = [] with open(args.corpus) as f: for line in f: line = line.strip() if line: entries.append(json.loads(line)) excluded = [e for e in entries if e.get("class", "replayable").startswith("excluded:")] replayable = [e for e in entries if e not in excluded] results = [] for entry in replayable: z = run_one(args.zsh, "-lc", entry["cmd"], args.worktree) n = run_one(args.nash, "-c", entry["cmd"], args.worktree) outcome = classify_outcome(z, n) expect = entry.get("expect", "match") if expect == "diverge": verdict = "SELFTEST_OK" if outcome == "DIVERGE" else "SELFTEST_BLIND" else: verdict = {"MATCH": "PASS", "STDERR_ONLY": "STDERR_ONLY", "DIVERGE": "DIVERGE"}[outcome] results.append({ "id": entry["id"], "cmd": entry["cmd"], "expect": expect, "verdict": verdict, "zsh": z, "nash": n, }) marker = {"PASS": ".", "STDERR_ONLY": "s", "DIVERGE": "X", "SELFTEST_OK": "+", "SELFTEST_BLIND": "!"}[verdict] print(marker, end="", flush=True) print() diverges = [r for r in results if r["verdict"] == "DIVERGE"] blind = [r for r in results if r["verdict"] == "SELFTEST_BLIND"] selftests = [r for r in results if r["expect"] == "diverge"] gated = [r for r in results if r["expect"] == "match"] passes = sum(1 for r in gated if r["verdict"] in ("PASS", "STDERR_ONLY")) parity = 100.0 * passes / len(gated) if gated else 0.0 print(f"\nreplayable: {len(gated)} parity: {passes}/{len(gated)} = {parity:.1f}%" f" self-tests: {len(selftests) - len(blind)}/{len(selftests)} caught") # Exclusions are part of the report, never silent (§10.6). for e in excluded: print(f" excluded [{e['class'].split(':', 1)[1]}]: {e['id']}: {e['cmd']!r}") for r in diverges + blind: print(f"\n{r['verdict']} {r['id']}: {r['cmd']!r}") print(f" zsh : code={r['zsh']['code']} stdout={r['zsh']['stdout']!r}") print(f" nash: code={r['nash']['code']} stdout={r['nash']['stdout']!r}") if r["zsh"]["fs"] != r["nash"]["fs"]: only = set(r["zsh"]["fs"]) ^ set(r["nash"]["fs"]) differs = { k for k in set(r["zsh"]["fs"]) & set(r["nash"]["fs"]) if r["zsh"]["fs"][k] != r["nash"]["fs"][k] } print(f" fs delta: only-one-side={sorted(only)} content-differs={sorted(differs)}") if args.json: with open(args.json, "w") as f: json.dump({"parity_percent": parity, "results": results, "excluded": excluded}, f, indent=1) ok = not diverges and not blind and selftests if not selftests: print("no self-test entries ran — the corpus must keep them (they prove the diff " "engine can see semantic divergence)") sys.exit(0 if ok else 1) if __name__ == "__main__": main()