Merge nucleic/lucid-north-quail-rnvt into dev
This commit is contained in:
+325
@@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Curate source prompts and build deterministic purpose-classifier splits."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Sequence
|
||||
|
||||
from purpose_data import (
|
||||
HARD_SLICES,
|
||||
LABELS,
|
||||
DataError,
|
||||
SourceRecord,
|
||||
curate_records,
|
||||
distribution,
|
||||
file_sha256,
|
||||
jsonl_bytes,
|
||||
load_classifiable_fixtures,
|
||||
load_sources,
|
||||
prompt_hash,
|
||||
split_records,
|
||||
write_json,
|
||||
write_jsonl,
|
||||
)
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
REPOSITORY_ROOT = SCRIPT_DIR.parent.parent
|
||||
DATA_DIR = SCRIPT_DIR / "data"
|
||||
GENERATION_MANIFEST = DATA_DIR / "generation-manifest.json"
|
||||
DEFAULT_FIXTURES = (
|
||||
REPOSITORY_ROOT
|
||||
/ "Tests"
|
||||
/ "NucleicCoreTests"
|
||||
/ "Fixtures"
|
||||
/ "purpose-prompts.json"
|
||||
)
|
||||
DEFAULT_OUTPUT_DIR = SCRIPT_DIR / ".artifacts" / "dataset-v1"
|
||||
DEFAULT_FROZEN_TEST = DATA_DIR / "frozen-test-v1.jsonl"
|
||||
DEFAULT_SPLIT_MANIFEST = DATA_DIR / "dataset-v1-manifest.json"
|
||||
DATASET_VERSION = "purpose-dataset-v1"
|
||||
DEFAULT_SEED = 0xC1A551F1
|
||||
|
||||
|
||||
def _relative(path: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(REPOSITORY_ROOT))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def default_source_paths() -> list[Path]:
|
||||
try:
|
||||
value = json.loads(GENERATION_MANIFEST.read_text(encoding="utf-8"))
|
||||
names = value["canonicalFiles"]
|
||||
except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError) as exc:
|
||||
raise DataError(f"{GENERATION_MANIFEST}: cannot read canonicalFiles: {exc}") from exc
|
||||
if not isinstance(names, list) or not names or not all(
|
||||
isinstance(name, str) and name for name in names
|
||||
):
|
||||
raise DataError(
|
||||
f"{GENERATION_MANIFEST}: canonicalFiles must be a non-empty string array"
|
||||
)
|
||||
return [(DATA_DIR / name).resolve() for name in names]
|
||||
|
||||
|
||||
def _fixture_counts(path: Path) -> tuple[int, int]:
|
||||
try:
|
||||
value = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise DataError(f"{path}: cannot read fixtures: {exc}") from exc
|
||||
if not isinstance(value, list):
|
||||
raise DataError(f"{path}: fixture root must be an array")
|
||||
classifiable = sum(
|
||||
isinstance(item, dict) and item.get("purpose") in LABELS for item in value
|
||||
)
|
||||
return len(value), classifiable
|
||||
|
||||
|
||||
def _sha256_bytes(value: bytes) -> str:
|
||||
return hashlib.sha256(value).hexdigest()
|
||||
|
||||
|
||||
def _records(values: Sequence[SourceRecord]) -> list[dict[str, Any]]:
|
||||
return [record.value for record in values]
|
||||
|
||||
|
||||
def _build_manifest(
|
||||
*,
|
||||
sources: Sequence[Path],
|
||||
source_record_count: int,
|
||||
curated_record_count: int,
|
||||
duplicate_counts: dict[str, int],
|
||||
splits: Any,
|
||||
fixtures_path: Path,
|
||||
total_fixture_count: int,
|
||||
classifiable_fixture_count: int,
|
||||
output_hashes: dict[str, str],
|
||||
seed: int,
|
||||
near_duplicate_threshold: float,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"schemaVersion": 1,
|
||||
"datasetVersion": DATASET_VERSION,
|
||||
"seed": seed,
|
||||
"ratios": {"train": 0.8, "validation": 0.1, "test": 0.1},
|
||||
"sources": [
|
||||
{
|
||||
"path": _relative(path),
|
||||
"sha256": file_sha256(path),
|
||||
"records": len(load_sources([path])),
|
||||
}
|
||||
for path in sources
|
||||
],
|
||||
"curation": {
|
||||
"inputRecords": source_record_count,
|
||||
"retainedRecords": curated_record_count,
|
||||
"excludedDuplicates": duplicate_counts,
|
||||
"nearDuplicateMethod": "word-trigram Jaccard after SimHash LSH candidate search",
|
||||
"nearDuplicateThreshold": near_duplicate_threshold,
|
||||
"vagueEvalPolicy": "validation/test only",
|
||||
},
|
||||
"frozenEval": {
|
||||
"syntheticPath": _relative(DEFAULT_FROZEN_TEST),
|
||||
"syntheticSha256": output_hashes["test"],
|
||||
"shippedFixturesPath": _relative(fixtures_path),
|
||||
"shippedFixturesSha256": file_sha256(fixtures_path),
|
||||
"shippedFixtureRecords": total_fixture_count,
|
||||
"classifiableShippedFixtureRecords": classifiable_fixture_count,
|
||||
"excludedGeneralFixtureRecords": (
|
||||
total_fixture_count - classifiable_fixture_count
|
||||
),
|
||||
"hardSliceDefinition": sorted(HARD_SLICES),
|
||||
},
|
||||
"splits": {
|
||||
"train": {
|
||||
"records": len(splits.train),
|
||||
"sha256": output_hashes["train"],
|
||||
"distribution": distribution(splits.train),
|
||||
},
|
||||
"validation": {
|
||||
"records": len(splits.validation),
|
||||
"sha256": output_hashes["validation"],
|
||||
"distribution": distribution(splits.validation),
|
||||
},
|
||||
"test": {
|
||||
"syntheticRecords": len(splits.test),
|
||||
"classifiableFixtureRecords": classifiable_fixture_count,
|
||||
"logicalRecords": splits.logical_test_count,
|
||||
"hardSyntheticRecords": sum(
|
||||
record.value["slice"] in HARD_SLICES for record in splits.test
|
||||
),
|
||||
"sha256": output_hashes["test"],
|
||||
"distribution": distribution(splits.test),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def prepare(
|
||||
*,
|
||||
sources: Sequence[Path],
|
||||
fixtures_path: Path,
|
||||
output_dir: Path,
|
||||
frozen_test_path: Path,
|
||||
manifest_path: Path,
|
||||
refresh_frozen_test: bool,
|
||||
seed: int,
|
||||
near_duplicate_threshold: float,
|
||||
) -> dict[str, Any]:
|
||||
records = load_sources(sources)
|
||||
fixtures = load_classifiable_fixtures(fixtures_path)
|
||||
total_fixture_count, classifiable_fixture_count = _fixture_counts(fixtures_path)
|
||||
if classifiable_fixture_count != len(fixtures):
|
||||
raise DataError("fixture accounting mismatch")
|
||||
|
||||
curated = curate_records(
|
||||
records,
|
||||
fixtures,
|
||||
near_duplicate_threshold=near_duplicate_threshold,
|
||||
)
|
||||
splits = split_records(
|
||||
curated.records,
|
||||
fixture_count=classifiable_fixture_count,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
train_values = _records(splits.train)
|
||||
validation_values = _records(splits.validation)
|
||||
test_values = _records(splits.test)
|
||||
candidate_frozen_test = jsonl_bytes(test_values)
|
||||
if refresh_frozen_test:
|
||||
frozen_test_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
frozen_test_path.write_bytes(candidate_frozen_test)
|
||||
elif not frozen_test_path.exists():
|
||||
raise DataError(
|
||||
f"{frozen_test_path}: frozen test is missing; review the candidate then run "
|
||||
"--refresh-frozen-test"
|
||||
)
|
||||
elif frozen_test_path.read_bytes() != candidate_frozen_test:
|
||||
raise DataError(
|
||||
f"{frozen_test_path}: deterministic test split changed; inspect source/seed "
|
||||
"changes and use --refresh-frozen-test only when intentionally versioning it"
|
||||
)
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
write_jsonl(output_dir / "train.jsonl", train_values)
|
||||
write_jsonl(output_dir / "validation.jsonl", validation_values)
|
||||
write_jsonl(output_dir / "test.jsonl", test_values)
|
||||
write_jsonl(
|
||||
output_dir / "exclusions.jsonl",
|
||||
(
|
||||
{
|
||||
"promptHash": prompt_hash(duplicate.dropped.value["prompt"]),
|
||||
"matchedPromptHash": duplicate.matched_prompt_hash,
|
||||
"source": _relative(duplicate.dropped.source),
|
||||
"line": duplicate.dropped.line,
|
||||
"kind": duplicate.kind,
|
||||
"similarity": round(duplicate.similarity, 6),
|
||||
}
|
||||
for duplicate in curated.duplicates
|
||||
),
|
||||
)
|
||||
|
||||
duplicate_counts: dict[str, int] = {}
|
||||
for duplicate in curated.duplicates:
|
||||
duplicate_counts[duplicate.kind] = duplicate_counts.get(duplicate.kind, 0) + 1
|
||||
output_hashes = {
|
||||
"train": _sha256_bytes(jsonl_bytes(train_values)),
|
||||
"validation": _sha256_bytes(jsonl_bytes(validation_values)),
|
||||
"test": _sha256_bytes(candidate_frozen_test),
|
||||
}
|
||||
manifest = _build_manifest(
|
||||
sources=sources,
|
||||
source_record_count=len(records),
|
||||
curated_record_count=len(curated.records),
|
||||
duplicate_counts=dict(sorted(duplicate_counts.items())),
|
||||
splits=splits,
|
||||
fixtures_path=fixtures_path,
|
||||
total_fixture_count=total_fixture_count,
|
||||
classifiable_fixture_count=classifiable_fixture_count,
|
||||
output_hashes=output_hashes,
|
||||
seed=seed,
|
||||
near_duplicate_threshold=near_duplicate_threshold,
|
||||
)
|
||||
# The frozen path can be overridden in tests or experiments.
|
||||
manifest["frozenEval"]["syntheticPath"] = _relative(frozen_test_path)
|
||||
write_json(output_dir / "manifest.json", manifest)
|
||||
if refresh_frozen_test:
|
||||
write_json(manifest_path, manifest)
|
||||
elif manifest_path.exists():
|
||||
existing = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if existing != manifest:
|
||||
raise DataError(
|
||||
f"{manifest_path}: split manifest changed; inspect and refresh the frozen "
|
||||
"test intentionally"
|
||||
)
|
||||
else:
|
||||
raise DataError(
|
||||
f"{manifest_path}: frozen split manifest is missing; use --refresh-frozen-test"
|
||||
)
|
||||
return manifest
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--source",
|
||||
action="append",
|
||||
type=Path,
|
||||
help="canonical source JSONL; repeat for multiple files (default: generation manifest)",
|
||||
)
|
||||
parser.add_argument("--fixtures", type=Path, default=DEFAULT_FIXTURES)
|
||||
parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR)
|
||||
parser.add_argument("--frozen-test", type=Path, default=DEFAULT_FROZEN_TEST)
|
||||
parser.add_argument("--manifest", type=Path, default=DEFAULT_SPLIT_MANIFEST)
|
||||
parser.add_argument(
|
||||
"--refresh-frozen-test",
|
||||
action="store_true",
|
||||
help="replace the versioned test split and manifest after intentional review",
|
||||
)
|
||||
parser.add_argument("--seed", type=int, default=DEFAULT_SEED)
|
||||
parser.add_argument("--near-duplicate-threshold", type=float, default=0.92)
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
try:
|
||||
sources = (
|
||||
[path.resolve() for path in args.source]
|
||||
if args.source
|
||||
else default_source_paths()
|
||||
)
|
||||
manifest = prepare(
|
||||
sources=sources,
|
||||
fixtures_path=args.fixtures.resolve(),
|
||||
output_dir=args.output_dir.resolve(),
|
||||
frozen_test_path=args.frozen_test.resolve(),
|
||||
manifest_path=args.manifest.resolve(),
|
||||
refresh_frozen_test=args.refresh_frozen_test,
|
||||
seed=args.seed,
|
||||
near_duplicate_threshold=args.near_duplicate_threshold,
|
||||
)
|
||||
except (DataError, OSError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
splits = manifest["splits"]
|
||||
print(
|
||||
f"Prepared {manifest['curation']['retainedRecords']} curated records: "
|
||||
f"{splits['train']['records']} train, "
|
||||
f"{splits['validation']['records']} validation, "
|
||||
f"{splits['test']['logicalRecords']} frozen test "
|
||||
f"({splits['test']['classifiableFixtureRecords']} shipped fixtures)."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user