nucleic-purpose-classifier/tests/test_review_contract.py

import csv
import sys
import tempfile
import unittest
from pathlib import Path


MODULE_DIR = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(MODULE_DIR))

import purpose_data
import review_contract


def record(index: int) -> purpose_data.SourceRecord:
    return purpose_data.SourceRecord(
        value={
            "prompt": f"Purpose review prompt {index}",
            "purpose": purpose_data.LABELS[index % len(purpose_data.LABELS)],
            "secondary": None,
            "mixed": False,
            "difficulty": 0.4,
            "slice": "core",
            "lang": "en",
        },
        source=Path("source.jsonl"),
        line=index + 1,
    )


def update_csv(path: Path, update):
    with path.open("r", encoding="utf-8", newline="") as handle:
        rows = list(csv.DictReader(handle))
    update(rows)
    with path.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(
            handle,
            fieldnames=review_contract.REVIEW_CSV_FIELDS,
        )
        writer.writeheader()
        writer.writerows(rows)


class HumanReviewContractTests(unittest.TestCase):
    def test_blank_review_reports_progress_without_claiming_completion(self):
        population = [record(index) for index in range(10)]
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / "review.csv"
            review_contract.write_review_csv(path, population)
            progress = review_contract.inspect_review_csv(path, population)

        self.assertEqual(0, progress.completed)
        self.assertEqual(10, progress.incomplete)
        self.assertEqual([], progress.decisions)

    def test_finalize_and_apply_accept_relabel_and_reject(self):
        population = [record(index) for index in range(10)]
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            csv_path = root / "review.csv"
            artifact_path = root / "human-review.json"
            review_contract.write_review_csv(csv_path, population)

            def complete(rows):
                for row in rows:
                    row["reviewStatus"] = "accept"
                rows[0]["reviewStatus"] = "relabel"
                rows[0]["reviewedPurpose"] = "writing"
                rows[0]["reviewNotes"] = "Primary intent is prose."
                rows[1]["reviewStatus"] = "reject"
                rows[1]["reviewNotes"] = "Prompt is not classifiable."
                rows[2]["reviewStatus"] = "relabel"
                rows[2]["reviewedSecondary"] = "review"
                rows[2]["reviewedSlice"] = "mixed"
                rows[2]["reviewNotes"] = "Two explicit intents."

            update_csv(csv_path, complete)
            artifact = review_contract.finalize_human_review(
                csv_path,
                artifact_path,
                population,
                dataset_version="test-v1",
                fraction=1.0,
                seed=7,
            )
            self.assertEqual(
                artifact,
                review_contract.finalize_human_review(
                    csv_path,
                    artifact_path,
                    population,
                    dataset_version="test-v1",
                    fraction=1.0,
                    seed=7,
                ),
            )
            result = review_contract.apply_completed_human_review(
                artifact_path,
                population,
                dataset_version="test-v1",
            )

        self.assertEqual(
            {"accepted": 7, "relabeled": 2, "rejected": 1},
            artifact["summary"],
        )
        self.assertEqual(9, len(result.records))
        by_prompt = {item.value["prompt"]: item.value for item in result.records}
        self.assertEqual("writing", by_prompt["Purpose review prompt 0"]["purpose"])
        self.assertNotIn("Purpose review prompt 1", by_prompt)
        self.assertEqual("review", by_prompt["Purpose review prompt 2"]["secondary"])
        self.assertTrue(by_prompt["Purpose review prompt 2"]["mixed"])
        self.assertEqual("mixed", by_prompt["Purpose review prompt 2"]["slice"])

    def test_finalize_fails_closed_when_rows_are_incomplete(self):
        population = [record(index) for index in range(5)]
        with tempfile.TemporaryDirectory() as directory:
            root = Path(directory)
            csv_path = root / "review.csv"
            review_contract.write_review_csv(csv_path, population)
            with self.assertRaisesRegex(
                purpose_data.DataError,
                "human review is incomplete: 0/5",
            ):
                review_contract.finalize_human_review(
                    csv_path,
                    root / "human-review.json",
                    population,
                    dataset_version="test-v1",
                    fraction=1.0,
                    seed=7,
                )

    def test_relabel_must_preserve_the_source_contract(self):
        population = [record(0)]
        with tempfile.TemporaryDirectory() as directory:
            path = Path(directory) / "review.csv"
            review_contract.write_review_csv(path, population)

            def invalidate(rows):
                rows[0]["reviewStatus"] = "relabel"
                rows[0]["reviewedSecondary"] = "review"
                rows[0]["reviewNotes"] = "Two intents."

            update_csv(path, invalidate)
            with self.assertRaisesRegex(
                purpose_data.DataError,
                "mixed slice and mixed field disagree",
            ):
                review_contract.inspect_review_csv(path, population)


if __name__ == "__main__":
    unittest.main()