From 14e04a4ed50a2d137cb1f4ab124c14bbbe9998be Mon Sep 17 00:00:00 2001 From: Nucleic Date: Sat, 1 Aug 2026 16:10:47 -0700 Subject: [PATCH] Merge nucleic/plucky-north-vole-sdna into dev --- README.md | 5 +++ label_swe_chat_prompts.py | 65 +++++++++++++++++++++++++--- tests/test_label_swe_chat_prompts.py | 19 ++++++++ 3 files changed, 82 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 9c58ef7..b3bd929 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,11 @@ run; both use Luna through subscription-backed `codex exec`, not an API key. The writes only the first message to canonical source JSONL; state and audit sidecars retain the response solely as a hash and source ID. +The labeler sends all remaining explanatory response prose to Luna. It replaces fenced +code and structured tool/reasoning payloads with size-marked placeholders, and fails on +oversized remaining prose rather than silently truncating it. The default response limit +is 48,000 characters; raise `--max-response-chars` only after inspecting an outlier. + ```bash ml/purpose-classifier/.venv/bin/python \ ml/purpose-classifier/label_swe_chat_prompts.py --limit-sessions 1 diff --git a/label_swe_chat_prompts.py b/label_swe_chat_prompts.py index edb29cb..8de36a5 100644 --- a/label_swe_chat_prompts.py +++ b/label_swe_chat_prompts.py @@ -12,6 +12,7 @@ import argparse import json import math import os +import re import subprocess import sys import tempfile @@ -28,6 +29,11 @@ DEFAULT_INPUT = SCRIPT_DIR / ".artifacts" / "swe-chat" / "candidates.jsonl" DEFAULT_OUTPUT = SCRIPT_DIR / ".artifacts" / "swe-chat" / "labeled-source.jsonl" MODEL = "gpt-5.6-luna" STATE_SCHEMA_VERSION = 1 +DEFAULT_MAX_RESPONSE_CHARS = 48_000 +FENCED_CODE_RE = re.compile(r"(?ms)^[ \t]*```[^\n]*\n.*?^[ \t]*```[ \t]*$") +TOOL_PAYLOAD_RE = re.compile( + r"(?is)<(?:tool_use|tool_result|function_call|function_calls|thinking|analysis)\b[^>]*>.*?" +) @dataclass(frozen=True) @@ -114,13 +120,54 @@ def response_schema(batch: Sequence[Candidate]) -> dict[str, Any]: return schema -def labeling_prompt(batch: Sequence[Candidate], max_chars: int) -> str: +def sanitize_agent_response(response: str) -> str: + """Keep complete explanatory prose while dropping code and tool payloads.""" + def replace(kind: str): + def inner(match: re.Match[str]) -> str: + return f"\n\n[{kind} omitted: {len(match.group(0)):,} characters]\n\n" + return inner + sanitized = FENCED_CODE_RE.sub(replace("fenced code"), response) + return TOOL_PAYLOAD_RE.sub(replace("tool or reasoning payload"), sanitized).strip() + + +def response_for_labeling(response: str, max_chars: int) -> str: + sanitized = sanitize_agent_response(response) + if not sanitized: + return "[agent response contained no prose after sanitization]" + if len(sanitized) > max_chars: + raise DataError( + f"sanitized agent response is {len(sanitized):,} characters, above the " + f"{max_chars:,}-character limit; inspect it or raise --max-response-chars " + "rather than truncating teacher context" + ) + return sanitized + + +def batches(candidates: Sequence[Candidate], *, batch_size: int, batch_chars: int, max_prompt_chars: int, max_response_chars: int) -> list[list[Candidate]]: + result: list[list[Candidate]] = [] + current: list[Candidate] = [] + current_chars = 0 + for candidate in candidates: + size = len(base.excerpt_for_labeling(candidate.prompt, max_prompt_chars)) + len(response_for_labeling(candidate.response, max_response_chars)) + if size > batch_chars: + raise DataError(f"{candidate.id}: sanitized prompt/response pair is {size:,} characters, above --batch-chars={batch_chars:,}") + if current and (len(current) >= batch_size or current_chars + size > batch_chars): + result.append(current) + current, current_chars = [], 0 + current.append(candidate) + current_chars += size + if current: + result.append(current) + return result + + +def labeling_prompt(batch: Sequence[Candidate], max_prompt_chars: int, max_response_chars: int) -> str: payload = { "items": [ { "id": candidate.id, - "first_message": base.excerpt_for_labeling(candidate.prompt, max_chars), - "first_agent_response": base.excerpt_for_labeling(candidate.response, max_chars), + "first_message": base.excerpt_for_labeling(candidate.prompt, max_prompt_chars), + "first_agent_response": response_for_labeling(candidate.response, max_response_chars), } for candidate in batch ] @@ -189,7 +236,7 @@ def validate_decisions(batch: Sequence[Candidate], response: Any) -> list[tuple[ def invoke_codex(args: argparse.Namespace, batch: Sequence[Candidate]) -> list[tuple[Candidate, dict[str, Any]]]: - prompt = labeling_prompt(batch, args.max_prompt_chars) + prompt = labeling_prompt(batch, args.max_prompt_chars, args.max_response_chars) last_error: Exception | None = None for attempt in range(1, args.max_attempts + 1): with tempfile.TemporaryDirectory(prefix="purpose-swe-label-") as directory: @@ -278,7 +325,10 @@ def run(args: argparse.Namespace) -> dict[str, int]: pending = [candidate for candidate in source if candidate.line.number not in states] if args.limit_sessions is not None: pending = pending[:args.limit_sessions] - batch_list = list(base.batches(pending, batch_size=args.batch_size, batch_chars=args.batch_chars, max_prompt_chars=args.max_prompt_chars)) + batch_list = batches( + pending, batch_size=args.batch_size, batch_chars=args.batch_chars, + max_prompt_chars=args.max_prompt_chars, max_response_chars=args.max_response_chars, + ) print(f"input={len(source)} resumed={len(states)} pending={len(pending)} batches={len(batch_list)} model={MODEL}", flush=True) for number, batch in enumerate(batch_list, 1): decisions = invoke_codex(args, batch) # type: ignore[arg-type] @@ -314,6 +364,7 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--batch-size", type=int, default=20) parser.add_argument("--batch-chars", type=int, default=80_000) parser.add_argument("--max-prompt-chars", type=int, default=24_000) + parser.add_argument("--max-response-chars", type=int, default=DEFAULT_MAX_RESPONSE_CHARS) parser.add_argument("--timeout-seconds", type=int, default=600) parser.add_argument("--max-attempts", type=int, default=3) parser.add_argument("--limit-sessions", type=int, help="bounded canary/dry-run session count") @@ -327,8 +378,8 @@ def main(argv: Sequence[str] | None = None) -> int: args.input, args.output = args.input.expanduser().resolve(), args.output.expanduser().resolve() args.state = args.state.expanduser().resolve() if args.state else args.output.with_name(f"{args.output.stem}.state.jsonl") args.audit = args.audit.expanduser().resolve() if args.audit else args.output.with_name(f"{args.output.stem}.audit.jsonl") - if args.batch_size <= 0 or args.batch_chars <= 0 or args.max_prompt_chars < 1_000 or args.timeout_seconds <= 0 or args.max_attempts <= 0 or (args.limit_sessions is not None and args.limit_sessions <= 0): - parser.error("batch sizes, timeout, attempts, and --limit-sessions must be positive; max prompt chars must be at least 1000") + if args.batch_size <= 0 or args.batch_chars <= 0 or args.max_prompt_chars < 1_000 or args.max_response_chars < 1_000 or args.timeout_seconds <= 0 or args.max_attempts <= 0 or (args.limit_sessions is not None and args.limit_sessions <= 0): + parser.error("batch sizes, timeout, attempts, and --limit-sessions must be positive; prompt/response limits must be at least 1000") try: metrics = run(args) except (DataError, OSError, ValueError, subprocess.SubprocessError) as error: diff --git a/tests/test_label_swe_chat_prompts.py b/tests/test_label_swe_chat_prompts.py index 11b5a4b..ca2d14c 100644 --- a/tests/test_label_swe_chat_prompts.py +++ b/tests/test_label_swe_chat_prompts.py @@ -64,6 +64,25 @@ class LabelSWEChatPromptsTests(unittest.TestCase): encoded = json.dumps(state) self.assertNotIn("It fails only on CI.", encoded) + def test_response_sanitizer_preserves_all_prose_and_removes_code_payloads(self): + response = ( + "I found the likely cause.\n\n" + "```swift\nlet secret = \"not teacher context\"\n```\n\n" + "The fix is to await the task.\n" + "{\"cmd\": \"rm -rf /\"}\n" + "Then rerun the focused test." + ) + sanitized = label_swe_chat_prompts.response_for_labeling(response, 2_000) + self.assertIn("I found the likely cause.", sanitized) + self.assertIn("The fix is to await the task.", sanitized) + self.assertIn("Then rerun the focused test.", sanitized) + self.assertNotIn("let secret", sanitized) + self.assertNotIn("rm -rf", sanitized) + + def test_response_sanitizer_refuses_to_truncate_prose(self): + with self.assertRaisesRegex(ValueError, "rather than truncating"): + label_swe_chat_prompts.response_for_labeling("x" * 1_001, 1_000) + if __name__ == "__main__": unittest.main()