#!/usr/bin/env python3 """Curate source prompts and build deterministic purpose-classifier splits.""" from __future__ import annotations import argparse import hashlib import json import sys from pathlib import Path from typing import Any, Sequence from purpose_data import ( HARD_SLICES, LABELS, DataError, SourceRecord, curate_records, distribution, file_sha256, jsonl_bytes, load_classifiable_fixtures, load_sources, prompt_hash, split_records, write_json, write_jsonl, ) SCRIPT_DIR = Path(__file__).resolve().parent REPOSITORY_ROOT = SCRIPT_DIR.parent.parent DATA_DIR = SCRIPT_DIR / "data" GENERATION_MANIFEST = DATA_DIR / "generation-manifest.json" DEFAULT_FIXTURES = ( REPOSITORY_ROOT / "Tests" / "NucleicCoreTests" / "Fixtures" / "purpose-prompts.json" ) DEFAULT_OUTPUT_DIR = SCRIPT_DIR / ".artifacts" / "dataset-v1" DEFAULT_FROZEN_TEST = DATA_DIR / "frozen-test-v1.jsonl" DEFAULT_SPLIT_MANIFEST = DATA_DIR / "dataset-v1-manifest.json" DATASET_VERSION = "purpose-dataset-v1" DEFAULT_SEED = 0xC1A551F1 def _relative(path: Path) -> str: try: return str(path.resolve().relative_to(REPOSITORY_ROOT)) except ValueError: return str(path.resolve()) def default_source_paths() -> list[Path]: try: value = json.loads(GENERATION_MANIFEST.read_text(encoding="utf-8")) names = value["canonicalFiles"] except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError) as exc: raise DataError(f"{GENERATION_MANIFEST}: cannot read canonicalFiles: {exc}") from exc if not isinstance(names, list) or not names or not all( isinstance(name, str) and name for name in names ): raise DataError( f"{GENERATION_MANIFEST}: canonicalFiles must be a non-empty string array" ) return [(DATA_DIR / name).resolve() for name in names] def _fixture_counts(path: Path) -> tuple[int, int]: try: value = json.loads(path.read_text(encoding="utf-8")) except (OSError, UnicodeError, json.JSONDecodeError) as exc: raise DataError(f"{path}: cannot read fixtures: {exc}") from exc if not isinstance(value, list): raise DataError(f"{path}: fixture root must be an array") classifiable = sum( isinstance(item, dict) and item.get("purpose") in LABELS for item in value ) return len(value), classifiable def _sha256_bytes(value: bytes) -> str: return hashlib.sha256(value).hexdigest() def _records(values: Sequence[SourceRecord]) -> list[dict[str, Any]]: return [record.value for record in values] def _build_manifest( *, sources: Sequence[Path], source_record_count: int, curated_record_count: int, duplicate_counts: dict[str, int], splits: Any, fixtures_path: Path, total_fixture_count: int, classifiable_fixture_count: int, output_hashes: dict[str, str], seed: int, near_duplicate_threshold: float, ) -> dict[str, Any]: return { "schemaVersion": 1, "datasetVersion": DATASET_VERSION, "seed": seed, "ratios": {"train": 0.8, "validation": 0.1, "test": 0.1}, "sources": [ { "path": _relative(path), "sha256": file_sha256(path), "records": len(load_sources([path])), } for path in sources ], "curation": { "inputRecords": source_record_count, "retainedRecords": curated_record_count, "excludedDuplicates": duplicate_counts, "nearDuplicateMethod": "word-trigram Jaccard after SimHash LSH candidate search", "nearDuplicateThreshold": near_duplicate_threshold, "vagueEvalPolicy": "validation/test only", }, "frozenEval": { "syntheticPath": _relative(DEFAULT_FROZEN_TEST), "syntheticSha256": output_hashes["test"], "shippedFixturesPath": _relative(fixtures_path), "shippedFixturesSha256": file_sha256(fixtures_path), "shippedFixtureRecords": total_fixture_count, "classifiableShippedFixtureRecords": classifiable_fixture_count, "excludedGeneralFixtureRecords": ( total_fixture_count - classifiable_fixture_count ), "hardSliceDefinition": sorted(HARD_SLICES), }, "splits": { "train": { "records": len(splits.train), "sha256": output_hashes["train"], "distribution": distribution(splits.train), }, "validation": { "records": len(splits.validation), "sha256": output_hashes["validation"], "distribution": distribution(splits.validation), }, "test": { "syntheticRecords": len(splits.test), "classifiableFixtureRecords": classifiable_fixture_count, "logicalRecords": splits.logical_test_count, "hardSyntheticRecords": sum( record.value["slice"] in HARD_SLICES for record in splits.test ), "sha256": output_hashes["test"], "distribution": distribution(splits.test), }, }, } def prepare( *, sources: Sequence[Path], fixtures_path: Path, output_dir: Path, frozen_test_path: Path, manifest_path: Path, refresh_frozen_test: bool, seed: int, near_duplicate_threshold: float, ) -> dict[str, Any]: records = load_sources(sources) fixtures = load_classifiable_fixtures(fixtures_path) total_fixture_count, classifiable_fixture_count = _fixture_counts(fixtures_path) if classifiable_fixture_count != len(fixtures): raise DataError("fixture accounting mismatch") curated = curate_records( records, fixtures, near_duplicate_threshold=near_duplicate_threshold, ) splits = split_records( curated.records, fixture_count=classifiable_fixture_count, seed=seed, ) train_values = _records(splits.train) validation_values = _records(splits.validation) test_values = _records(splits.test) candidate_frozen_test = jsonl_bytes(test_values) if refresh_frozen_test: frozen_test_path.parent.mkdir(parents=True, exist_ok=True) frozen_test_path.write_bytes(candidate_frozen_test) elif not frozen_test_path.exists(): raise DataError( f"{frozen_test_path}: frozen test is missing; review the candidate then run " "--refresh-frozen-test" ) elif frozen_test_path.read_bytes() != candidate_frozen_test: raise DataError( f"{frozen_test_path}: deterministic test split changed; inspect source/seed " "changes and use --refresh-frozen-test only when intentionally versioning it" ) output_dir.mkdir(parents=True, exist_ok=True) write_jsonl(output_dir / "train.jsonl", train_values) write_jsonl(output_dir / "validation.jsonl", validation_values) write_jsonl(output_dir / "test.jsonl", test_values) write_jsonl( output_dir / "exclusions.jsonl", ( { "promptHash": prompt_hash(duplicate.dropped.value["prompt"]), "matchedPromptHash": duplicate.matched_prompt_hash, "source": _relative(duplicate.dropped.source), "line": duplicate.dropped.line, "kind": duplicate.kind, "similarity": round(duplicate.similarity, 6), } for duplicate in curated.duplicates ), ) duplicate_counts: dict[str, int] = {} for duplicate in curated.duplicates: duplicate_counts[duplicate.kind] = duplicate_counts.get(duplicate.kind, 0) + 1 output_hashes = { "train": _sha256_bytes(jsonl_bytes(train_values)), "validation": _sha256_bytes(jsonl_bytes(validation_values)), "test": _sha256_bytes(candidate_frozen_test), } manifest = _build_manifest( sources=sources, source_record_count=len(records), curated_record_count=len(curated.records), duplicate_counts=dict(sorted(duplicate_counts.items())), splits=splits, fixtures_path=fixtures_path, total_fixture_count=total_fixture_count, classifiable_fixture_count=classifiable_fixture_count, output_hashes=output_hashes, seed=seed, near_duplicate_threshold=near_duplicate_threshold, ) # The frozen path can be overridden in tests or experiments. manifest["frozenEval"]["syntheticPath"] = _relative(frozen_test_path) write_json(output_dir / "manifest.json", manifest) if refresh_frozen_test: write_json(manifest_path, manifest) elif manifest_path.exists(): existing = json.loads(manifest_path.read_text(encoding="utf-8")) if existing != manifest: raise DataError( f"{manifest_path}: split manifest changed; inspect and refresh the frozen " "test intentionally" ) else: raise DataError( f"{manifest_path}: frozen split manifest is missing; use --refresh-frozen-test" ) return manifest def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--source", action="append", type=Path, help="canonical source JSONL; repeat for multiple files (default: generation manifest)", ) parser.add_argument("--fixtures", type=Path, default=DEFAULT_FIXTURES) parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) parser.add_argument("--frozen-test", type=Path, default=DEFAULT_FROZEN_TEST) parser.add_argument("--manifest", type=Path, default=DEFAULT_SPLIT_MANIFEST) parser.add_argument( "--refresh-frozen-test", action="store_true", help="replace the versioned test split and manifest after intentional review", ) parser.add_argument("--seed", type=int, default=DEFAULT_SEED) parser.add_argument("--near-duplicate-threshold", type=float, default=0.92) return parser def main(argv: Sequence[str] | None = None) -> int: args = build_parser().parse_args(argv) try: sources = ( [path.resolve() for path in args.source] if args.source else default_source_paths() ) manifest = prepare( sources=sources, fixtures_path=args.fixtures.resolve(), output_dir=args.output_dir.resolve(), frozen_test_path=args.frozen_test.resolve(), manifest_path=args.manifest.resolve(), refresh_frozen_test=args.refresh_frozen_test, seed=args.seed, near_duplicate_threshold=args.near_duplicate_threshold, ) except (DataError, OSError, UnicodeError, json.JSONDecodeError) as exc: print(f"error: {exc}", file=sys.stderr) return 1 splits = manifest["splits"] print( f"Prepared {manifest['curation']['retainedRecords']} curated records: " f"{splits['train']['records']} train, " f"{splits['validation']['records']} validation, " f"{splits['test']['logicalRecords']} frozen test " f"({splits['test']['classifiableFixtureRecords']} shipped fixtures)." ) return 0 if __name__ == "__main__": raise SystemExit(main())