Merge nucleic/plucky-north-vole-sdna into dev

README.md

@@ -83,6 +83,11 @@ run; both use Luna through subscription-backed `codex exec`, not an API key. The
 writes only the first message to canonical source JSONL; state and audit sidecars retain
 the response solely as a hash and source ID.
 
+The labeler sends all remaining explanatory response prose to Luna. It replaces fenced
+code and structured tool/reasoning payloads with size-marked placeholders, and fails on
+oversized remaining prose rather than silently truncating it. The default response limit
+is 48,000 characters; raise `--max-response-chars` only after inspecting an outlier.
+
 ```bash
 ml/purpose-classifier/.venv/bin/python \
   ml/purpose-classifier/label_swe_chat_prompts.py --limit-sessions 1

label_swe_chat_prompts.py

@@ -12,6 +12,7 @@ import argparse
 import json
 import math
 import os
+import re
 import subprocess
 import sys
 import tempfile
@@ -28,6 +29,11 @@ DEFAULT_INPUT = SCRIPT_DIR / ".artifacts" / "swe-chat" / "candidates.jsonl"
 DEFAULT_OUTPUT = SCRIPT_DIR / ".artifacts" / "swe-chat" / "labeled-source.jsonl"
 MODEL = "gpt-5.6-luna"
 STATE_SCHEMA_VERSION = 1
+DEFAULT_MAX_RESPONSE_CHARS = 48_000
+FENCED_CODE_RE = re.compile(r"(?ms)^[ \t]*```[^\n]*\n.*?^[ \t]*```[ \t]*$")
+TOOL_PAYLOAD_RE = re.compile(
+    r"(?is)<(?:tool_use|tool_result|function_call|function_calls|thinking|analysis)\b[^>]*>.*?</(?:tool_use|tool_result|function_call|function_calls|thinking|analysis)>"
+)
 
 
 @dataclass(frozen=True)
@@ -114,13 +120,54 @@ def response_schema(batch: Sequence[Candidate]) -> dict[str, Any]:
     return schema
 
 
-def labeling_prompt(batch: Sequence[Candidate], max_chars: int) -> str:
+def sanitize_agent_response(response: str) -> str:
+    """Keep complete explanatory prose while dropping code and tool payloads."""
+    def replace(kind: str):
+        def inner(match: re.Match[str]) -> str:
+            return f"\n\n[{kind} omitted: {len(match.group(0)):,} characters]\n\n"
+        return inner
+    sanitized = FENCED_CODE_RE.sub(replace("fenced code"), response)
+    return TOOL_PAYLOAD_RE.sub(replace("tool or reasoning payload"), sanitized).strip()
+
+
+def response_for_labeling(response: str, max_chars: int) -> str:
+    sanitized = sanitize_agent_response(response)
+    if not sanitized:
+        return "[agent response contained no prose after sanitization]"
+    if len(sanitized) > max_chars:
+        raise DataError(
+            f"sanitized agent response is {len(sanitized):,} characters, above the "
+            f"{max_chars:,}-character limit; inspect it or raise --max-response-chars "
+            "rather than truncating teacher context"
+        )
+    return sanitized
+
+
+def batches(candidates: Sequence[Candidate], *, batch_size: int, batch_chars: int, max_prompt_chars: int, max_response_chars: int) -> list[list[Candidate]]:
+    result: list[list[Candidate]] = []
+    current: list[Candidate] = []
+    current_chars = 0
+    for candidate in candidates:
+        size = len(base.excerpt_for_labeling(candidate.prompt, max_prompt_chars)) + len(response_for_labeling(candidate.response, max_response_chars))
+        if size > batch_chars:
+            raise DataError(f"{candidate.id}: sanitized prompt/response pair is {size:,} characters, above --batch-chars={batch_chars:,}")
+        if current and (len(current) >= batch_size or current_chars + size > batch_chars):
+            result.append(current)
+            current, current_chars = [], 0
+        current.append(candidate)
+        current_chars += size
+    if current:
+        result.append(current)
+    return result
+
+
+def labeling_prompt(batch: Sequence[Candidate], max_prompt_chars: int, max_response_chars: int) -> str:
     payload = {
         "items": [
             {
                 "id": candidate.id,
-                "first_message": base.excerpt_for_labeling(candidate.prompt, max_chars),
+                "first_message": base.excerpt_for_labeling(candidate.prompt, max_prompt_chars),
-                "first_agent_response": base.excerpt_for_labeling(candidate.response, max_chars),
+                "first_agent_response": response_for_labeling(candidate.response, max_response_chars),
             }
             for candidate in batch
         ]
@@ -189,7 +236,7 @@ def validate_decisions(batch: Sequence[Candidate], response: Any) -> list[tuple[
 
 
 def invoke_codex(args: argparse.Namespace, batch: Sequence[Candidate]) -> list[tuple[Candidate, dict[str, Any]]]:
-    prompt = labeling_prompt(batch, args.max_prompt_chars)
+    prompt = labeling_prompt(batch, args.max_prompt_chars, args.max_response_chars)
     last_error: Exception | None = None
     for attempt in range(1, args.max_attempts + 1):
         with tempfile.TemporaryDirectory(prefix="purpose-swe-label-") as directory:
@@ -278,7 +325,10 @@ def run(args: argparse.Namespace) -> dict[str, int]:
     pending = [candidate for candidate in source if candidate.line.number not in states]
     if args.limit_sessions is not None:
         pending = pending[:args.limit_sessions]
-    batch_list = list(base.batches(pending, batch_size=args.batch_size, batch_chars=args.batch_chars, max_prompt_chars=args.max_prompt_chars))
+    batch_list = batches(
+        pending, batch_size=args.batch_size, batch_chars=args.batch_chars,
+        max_prompt_chars=args.max_prompt_chars, max_response_chars=args.max_response_chars,
+    )
     print(f"input={len(source)} resumed={len(states)} pending={len(pending)} batches={len(batch_list)} model={MODEL}", flush=True)
     for number, batch in enumerate(batch_list, 1):
         decisions = invoke_codex(args, batch)  # type: ignore[arg-type]
@@ -314,6 +364,7 @@ def build_parser() -> argparse.ArgumentParser:
     parser.add_argument("--batch-size", type=int, default=20)
     parser.add_argument("--batch-chars", type=int, default=80_000)
     parser.add_argument("--max-prompt-chars", type=int, default=24_000)
+    parser.add_argument("--max-response-chars", type=int, default=DEFAULT_MAX_RESPONSE_CHARS)
     parser.add_argument("--timeout-seconds", type=int, default=600)
     parser.add_argument("--max-attempts", type=int, default=3)
     parser.add_argument("--limit-sessions", type=int, help="bounded canary/dry-run session count")
@@ -327,8 +378,8 @@ def main(argv: Sequence[str] | None = None) -> int:
     args.input, args.output = args.input.expanduser().resolve(), args.output.expanduser().resolve()
     args.state = args.state.expanduser().resolve() if args.state else args.output.with_name(f"{args.output.stem}.state.jsonl")
     args.audit = args.audit.expanduser().resolve() if args.audit else args.output.with_name(f"{args.output.stem}.audit.jsonl")
-    if args.batch_size <= 0 or args.batch_chars <= 0 or args.max_prompt_chars < 1_000 or args.timeout_seconds <= 0 or args.max_attempts <= 0 or (args.limit_sessions is not None and args.limit_sessions <= 0):
+    if args.batch_size <= 0 or args.batch_chars <= 0 or args.max_prompt_chars < 1_000 or args.max_response_chars < 1_000 or args.timeout_seconds <= 0 or args.max_attempts <= 0 or (args.limit_sessions is not None and args.limit_sessions <= 0):
-        parser.error("batch sizes, timeout, attempts, and --limit-sessions must be positive; max prompt chars must be at least 1000")
+        parser.error("batch sizes, timeout, attempts, and --limit-sessions must be positive; prompt/response limits must be at least 1000")
     try:
         metrics = run(args)
     except (DataError, OSError, ValueError, subprocess.SubprocessError) as error:

tests/test_label_swe_chat_prompts.py

@@ -64,6 +64,25 @@ class LabelSWEChatPromptsTests(unittest.TestCase):
         encoded = json.dumps(state)
         self.assertNotIn("It fails only on CI.", encoded)
 
+    def test_response_sanitizer_preserves_all_prose_and_removes_code_payloads(self):
+        response = (
+            "I found the likely cause.\n\n"
+            "```swift\nlet secret = \"not teacher context\"\n```\n\n"
+            "The fix is to await the task.\n"
+            "<tool_use>{\"cmd\": \"rm -rf /\"}</tool_use>\n"
+            "Then rerun the focused test."
+        )
+        sanitized = label_swe_chat_prompts.response_for_labeling(response, 2_000)
+        self.assertIn("I found the likely cause.", sanitized)
+        self.assertIn("The fix is to await the task.", sanitized)
+        self.assertIn("Then rerun the focused test.", sanitized)
+        self.assertNotIn("let secret", sanitized)
+        self.assertNotIn("rm -rf", sanitized)
+
+    def test_response_sanitizer_refuses_to_truncate_prose(self):
+        with self.assertRaisesRegex(ValueError, "rather than truncating"):
+            label_swe_chat_prompts.response_for_labeling("x" * 1_001, 1_000)
+
 
 if __name__ == "__main__":
     unittest.main()