Merge nucleic/dapper-feather-lynx-w4wa into dev
This commit is contained in:
Executable
+213
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Differential replay harness for the macOS HOST surface (docs/NASH.md §10.6 — the M5.5 gate).
|
||||
|
||||
The host risk is *semantic divergence*, not parse failure: a command that parses under nash but
|
||||
means something different under zsh (unquoted word splitting, no-match globbing) runs to
|
||||
completion with different results and no fallback event. So this harness replays each corpus
|
||||
command under BOTH `zsh -lc` (today's host shell) and `nash -c` (the M5.5 flip: login sourcing is
|
||||
replaced by the LoginShellEnv snapshot, hence `-c`) in identical disposable workspaces, then
|
||||
diffs exit code, stdout, AND the resulting file-system tree. stderr is compared separately and
|
||||
never counts against parity (error wording legitimately differs).
|
||||
|
||||
Corpus entries (host-corpus.jsonl, one JSON object per line):
|
||||
{"id": "...", "cmd": "...", # a replayable command
|
||||
"class": "replayable" | "excluded:<reason>", # exclusions are REPORTED, never silent
|
||||
"expect": "match" | "diverge"} # 'diverge' = harness SELF-TEST entry: a known
|
||||
# zsh≠bash-family semantic difference the
|
||||
# diff engine MUST catch (else the engine is
|
||||
# blind and the gate is meaningless)
|
||||
|
||||
Workspaces: a synthetic fixture tree by default; pass --worktree PATH to clone a real project
|
||||
tree instead (cp -R into the scratch dir) so toolchain commands (`swift build`, `make`) replay
|
||||
against real inputs — §10.6's "disposable clone".
|
||||
|
||||
Verifying the harness before a darwin nash exists: `--nash /bin/bash` is a faithful stand-in for
|
||||
the divergence semantics under test (bash word-splits and passes unmatched globs through exactly
|
||||
as nash's brush core does; zsh does neither), so the self-test entries must report DIVERGE and
|
||||
real entries' verdicts are meaningful. The real gate run uses the built nash.
|
||||
|
||||
Exit status: 0 iff every replayable expect-match entry matched AND every expect-diverge
|
||||
self-test diverged AND at least one self-test ran.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
FIXTURES = {
|
||||
"a.txt": "first line\nsecond line\nthird line\n",
|
||||
"b.txt": "banana\napple\nbanana\ncherry\napple\n",
|
||||
"data.csv": "name,team,score\nalice,red,10\nbob,blue,7\ncarol,red,9\ndave,blue,7\n",
|
||||
"logs/app.log": (
|
||||
"2026-01-01 INFO boot ok\n"
|
||||
"2026-01-01 ERROR disk full\n"
|
||||
"2026-01-01 INFO retry\n"
|
||||
"2026-01-01 WARN slow\n"
|
||||
"2026-01-01 ERROR net down\n"
|
||||
"2026-01-01 INFO done\n"
|
||||
),
|
||||
"src/main.py": "print('main')\n",
|
||||
"src/util.py": "def add(a, b):\n return a + b\n",
|
||||
"README.md": "# Sample\n",
|
||||
"Makefile": "check:\n\t@echo make-ran\n",
|
||||
}
|
||||
|
||||
TIMEOUT_S = 60
|
||||
|
||||
|
||||
def seed(workdir, worktree=None):
|
||||
if worktree:
|
||||
shutil.copytree(worktree, workdir, symlinks=True, dirs_exist_ok=True)
|
||||
return
|
||||
for rel, content in FIXTURES.items():
|
||||
path = os.path.join(workdir, rel)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with open(path, "w") as f:
|
||||
f.write(content)
|
||||
|
||||
|
||||
def snapshot(workdir):
|
||||
"""Relative path -> sha256 of contents, for every file in the tree."""
|
||||
out = {}
|
||||
for root, _dirs, files in os.walk(workdir):
|
||||
for name in files:
|
||||
path = os.path.join(root, name)
|
||||
rel = os.path.relpath(path, workdir)
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
out[rel] = hashlib.sha256(f.read()).hexdigest()
|
||||
except OSError:
|
||||
out[rel] = "<unreadable>"
|
||||
return out
|
||||
|
||||
|
||||
def run_one(shell, mode, cmd, worktree=None):
|
||||
parent = tempfile.mkdtemp(prefix="nash-hostdiff-")
|
||||
workdir = os.path.join(parent, "workspace")
|
||||
os.makedirs(workdir, exist_ok=True)
|
||||
seed(workdir, worktree)
|
||||
# A pinned, minimal env on purpose: the point of §7.7.2 is that the flip must NOT depend on
|
||||
# what login sourcing would add — a command that only works under `zsh -lc`'s rc-sourced env
|
||||
# shows up here as a divergence to investigate, which is the gate doing its job.
|
||||
env = {
|
||||
"PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
|
||||
"HOME": workdir,
|
||||
"LC_ALL": "C",
|
||||
"LANG": "C",
|
||||
"TERM": "dumb",
|
||||
"SHELL": shell,
|
||||
}
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[shell, mode, cmd], cwd=workdir, env=env, capture_output=True, timeout=TIMEOUT_S
|
||||
)
|
||||
code, out, err = proc.returncode, proc.stdout, proc.stderr
|
||||
except subprocess.TimeoutExpired:
|
||||
code, out, err = "TIMEOUT", b"", b""
|
||||
fs = snapshot(workdir)
|
||||
shutil.rmtree(parent, ignore_errors=True)
|
||||
|
||||
def norm(b):
|
||||
text = b.decode("utf-8", "replace")
|
||||
return text.replace(workdir, "__WORK__").replace(parent, "__TMP__")
|
||||
|
||||
return {"code": code, "stdout": norm(out), "stderr": norm(err), "fs": fs}
|
||||
|
||||
|
||||
def classify_outcome(z, n):
|
||||
core_equal = (
|
||||
z["code"] == n["code"] and z["stdout"] == n["stdout"] and z["fs"] == n["fs"]
|
||||
)
|
||||
if core_equal and z["stderr"] == n["stderr"]:
|
||||
return "MATCH"
|
||||
if core_equal:
|
||||
return "STDERR_ONLY"
|
||||
return "DIVERGE"
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--nash", default=os.environ.get("NASH_BIN", "nash"),
|
||||
help="nash binary (or /bin/bash as a semantics stand-in pre-nash)")
|
||||
ap.add_argument("--zsh", default="/bin/zsh")
|
||||
ap.add_argument(
|
||||
"--corpus",
|
||||
default=os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "host-corpus.jsonl"),
|
||||
)
|
||||
ap.add_argument("--worktree", help="clone this tree as the workspace (disposable copy)")
|
||||
ap.add_argument("--json", help="also write the full report to this path")
|
||||
args = ap.parse_args()
|
||||
|
||||
entries = []
|
||||
with open(args.corpus) as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
entries.append(json.loads(line))
|
||||
|
||||
excluded = [e for e in entries if e.get("class", "replayable").startswith("excluded:")]
|
||||
replayable = [e for e in entries if e not in excluded]
|
||||
|
||||
results = []
|
||||
for entry in replayable:
|
||||
z = run_one(args.zsh, "-lc", entry["cmd"], args.worktree)
|
||||
n = run_one(args.nash, "-c", entry["cmd"], args.worktree)
|
||||
outcome = classify_outcome(z, n)
|
||||
expect = entry.get("expect", "match")
|
||||
if expect == "diverge":
|
||||
verdict = "SELFTEST_OK" if outcome == "DIVERGE" else "SELFTEST_BLIND"
|
||||
else:
|
||||
verdict = {"MATCH": "PASS", "STDERR_ONLY": "STDERR_ONLY", "DIVERGE": "DIVERGE"}[outcome]
|
||||
results.append({
|
||||
"id": entry["id"], "cmd": entry["cmd"], "expect": expect,
|
||||
"verdict": verdict, "zsh": z, "nash": n,
|
||||
})
|
||||
marker = {"PASS": ".", "STDERR_ONLY": "s", "DIVERGE": "X",
|
||||
"SELFTEST_OK": "+", "SELFTEST_BLIND": "!"}[verdict]
|
||||
print(marker, end="", flush=True)
|
||||
print()
|
||||
|
||||
diverges = [r for r in results if r["verdict"] == "DIVERGE"]
|
||||
blind = [r for r in results if r["verdict"] == "SELFTEST_BLIND"]
|
||||
selftests = [r for r in results if r["expect"] == "diverge"]
|
||||
gated = [r for r in results if r["expect"] == "match"]
|
||||
passes = sum(1 for r in gated if r["verdict"] in ("PASS", "STDERR_ONLY"))
|
||||
parity = 100.0 * passes / len(gated) if gated else 0.0
|
||||
|
||||
print(f"\nreplayable: {len(gated)} parity: {passes}/{len(gated)} = {parity:.1f}%"
|
||||
f" self-tests: {len(selftests) - len(blind)}/{len(selftests)} caught")
|
||||
# Exclusions are part of the report, never silent (§10.6).
|
||||
for e in excluded:
|
||||
print(f" excluded [{e['class'].split(':', 1)[1]}]: {e['id']}: {e['cmd']!r}")
|
||||
for r in diverges + blind:
|
||||
print(f"\n{r['verdict']} {r['id']}: {r['cmd']!r}")
|
||||
print(f" zsh : code={r['zsh']['code']} stdout={r['zsh']['stdout']!r}")
|
||||
print(f" nash: code={r['nash']['code']} stdout={r['nash']['stdout']!r}")
|
||||
if r["zsh"]["fs"] != r["nash"]["fs"]:
|
||||
only = set(r["zsh"]["fs"]) ^ set(r["nash"]["fs"])
|
||||
differs = {
|
||||
k for k in set(r["zsh"]["fs"]) & set(r["nash"]["fs"])
|
||||
if r["zsh"]["fs"][k] != r["nash"]["fs"][k]
|
||||
}
|
||||
print(f" fs delta: only-one-side={sorted(only)} content-differs={sorted(differs)}")
|
||||
|
||||
if args.json:
|
||||
with open(args.json, "w") as f:
|
||||
json.dump({"parity_percent": parity, "results": results,
|
||||
"excluded": excluded}, f, indent=1)
|
||||
|
||||
ok = not diverges and not blind and selftests
|
||||
if not selftests:
|
||||
print("no self-test entries ran — the corpus must keep them (they prove the diff "
|
||||
"engine can see semantic divergence)")
|
||||
sys.exit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user