Merge nucleic/lucid-north-quail-rnvt into dev
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(MODULE_DIR))
|
||||
|
||||
import prepare_data
|
||||
import purpose_data
|
||||
|
||||
|
||||
def example(index: int):
|
||||
return {
|
||||
"prompt": f"Implement sample endpoint number {index} with stable pagination",
|
||||
"purpose": purpose_data.LABELS[index % len(purpose_data.LABELS)],
|
||||
"secondary": None,
|
||||
"mixed": False,
|
||||
"difficulty": 0.4,
|
||||
"slice": "vague-eval" if index < 5 else "core",
|
||||
"lang": "en",
|
||||
}
|
||||
|
||||
|
||||
class PrepareIntegrationTests(unittest.TestCase):
|
||||
def test_refresh_then_verify_frozen_split(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
source = root / "source.jsonl"
|
||||
purpose_data.write_jsonl(source, (example(index) for index in range(80)))
|
||||
fixtures = root / "fixtures.json"
|
||||
fixtures.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{"prompt": "Plan the cache migration", "purpose": "planning"},
|
||||
{"prompt": "Anything else?", "purpose": "general"},
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
output = root / "output"
|
||||
frozen = root / "frozen.jsonl"
|
||||
manifest = root / "manifest.json"
|
||||
|
||||
first = prepare_data.prepare(
|
||||
sources=[source],
|
||||
fixtures_path=fixtures,
|
||||
output_dir=output,
|
||||
frozen_test_path=frozen,
|
||||
manifest_path=manifest,
|
||||
refresh_frozen_test=True,
|
||||
seed=23,
|
||||
near_duplicate_threshold=0.92,
|
||||
)
|
||||
second = prepare_data.prepare(
|
||||
sources=[source],
|
||||
fixtures_path=fixtures,
|
||||
output_dir=output,
|
||||
frozen_test_path=frozen,
|
||||
manifest_path=manifest,
|
||||
refresh_frozen_test=False,
|
||||
seed=23,
|
||||
near_duplicate_threshold=0.92,
|
||||
)
|
||||
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(65, first["splits"]["train"]["records"])
|
||||
self.assertEqual(8, first["splits"]["validation"]["records"])
|
||||
self.assertEqual(8, first["splits"]["test"]["logicalRecords"])
|
||||
train = purpose_data.load_jsonl(output / "train.jsonl")
|
||||
self.assertFalse(any(row["slice"] == "vague-eval" for row in train))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,120 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(MODULE_DIR))
|
||||
|
||||
import purpose_data
|
||||
|
||||
|
||||
def example(index: int, **overrides):
|
||||
value = {
|
||||
"prompt": f"Implement sample endpoint number {index} with stable pagination",
|
||||
"purpose": purpose_data.LABELS[index % len(purpose_data.LABELS)],
|
||||
"secondary": None,
|
||||
"mixed": False,
|
||||
"difficulty": 0.4,
|
||||
"slice": "core",
|
||||
"lang": "en",
|
||||
}
|
||||
value.update(overrides)
|
||||
return value
|
||||
|
||||
|
||||
def source(value, line=1):
|
||||
return purpose_data.SourceRecord(value, Path("source.jsonl"), line)
|
||||
|
||||
|
||||
class NormalizationTests(unittest.TestCase):
|
||||
def test_normalization_matches_runtime_whitespace_contract(self):
|
||||
self.assertEqual(
|
||||
"Café deploy now",
|
||||
purpose_data.normalize_prompt(" Cafe\u0301\tdeploy\nnow "),
|
||||
)
|
||||
self.assertEqual(
|
||||
purpose_data.normalized_key("FIX spacing"),
|
||||
purpose_data.normalized_key(" fix spacing "),
|
||||
)
|
||||
|
||||
|
||||
class CurationTests(unittest.TestCase):
|
||||
def test_excludes_exact_fixture_overlap(self):
|
||||
record = source(example(0, prompt="Make the toolbar nicer", purpose="frontendImpl"))
|
||||
result = purpose_data.curate_records(
|
||||
[record],
|
||||
[{"prompt": " make the toolbar nicer ", "purpose": "frontendImpl"}],
|
||||
)
|
||||
self.assertEqual([], result.records)
|
||||
self.assertEqual("exact", result.duplicates[0].kind)
|
||||
|
||||
def test_excludes_high_overlap_generated_template(self):
|
||||
words = [f"token{index}" for index in range(100)]
|
||||
first = " ".join(words)
|
||||
words[50] = "replacement"
|
||||
second = " ".join(words)
|
||||
result = purpose_data.curate_records(
|
||||
[
|
||||
source(example(0, prompt=first, purpose="backendImpl"), 1),
|
||||
source(example(1, prompt=second, purpose="backendImpl"), 2),
|
||||
],
|
||||
[],
|
||||
near_duplicate_threshold=0.92,
|
||||
)
|
||||
self.assertEqual(1, len(result.records))
|
||||
self.assertEqual(1, len(result.duplicates))
|
||||
self.assertEqual("near", result.duplicates[0].kind)
|
||||
self.assertGreaterEqual(result.duplicates[0].similarity, 0.92)
|
||||
|
||||
def test_near_duplicate_label_conflict_requires_review(self):
|
||||
words = [f"token{index}" for index in range(100)]
|
||||
first = " ".join(words)
|
||||
words[50] = "replacement"
|
||||
second = " ".join(words)
|
||||
with self.assertRaisesRegex(
|
||||
purpose_data.DataError, "label conflicts require review"
|
||||
):
|
||||
purpose_data.curate_records(
|
||||
[
|
||||
source(example(0, prompt=first, purpose="backendImpl"), 1),
|
||||
source(example(1, prompt=second, purpose="writing"), 2),
|
||||
],
|
||||
[],
|
||||
near_duplicate_threshold=0.92,
|
||||
)
|
||||
|
||||
|
||||
class SplitTests(unittest.TestCase):
|
||||
def test_split_is_deterministic_stratified_and_keeps_vague_out_of_train(self):
|
||||
records = []
|
||||
for index in range(1_000):
|
||||
slice_name = "vague-eval" if index < 50 else (
|
||||
"boundary" if index % 5 == 0 else "core"
|
||||
)
|
||||
records.append(source(example(index, slice=slice_name), index + 1))
|
||||
|
||||
first = purpose_data.split_records(records, fixture_count=10, seed=17)
|
||||
second = purpose_data.split_records(records, fixture_count=10, seed=17)
|
||||
|
||||
self.assertEqual(
|
||||
[row.value["prompt"] for row in first.train],
|
||||
[row.value["prompt"] for row in second.train],
|
||||
)
|
||||
self.assertEqual(808, len(first.train))
|
||||
self.assertEqual(101, len(first.validation))
|
||||
self.assertEqual(101, first.logical_test_count)
|
||||
self.assertFalse(
|
||||
any(row.value["slice"] == "vague-eval" for row in first.train)
|
||||
)
|
||||
split_prompts = [
|
||||
{row.value["prompt"] for row in split}
|
||||
for split in (first.train, first.validation, first.test)
|
||||
]
|
||||
self.assertFalse(split_prompts[0] & split_prompts[1])
|
||||
self.assertFalse(split_prompts[0] & split_prompts[2])
|
||||
self.assertFalse(split_prompts[1] & split_prompts[2])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,46 @@
|
||||
import sys
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
MODULE_DIR = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(MODULE_DIR))
|
||||
|
||||
import train
|
||||
|
||||
|
||||
class MetricsTests(unittest.TestCase):
|
||||
def test_classification_metrics_include_every_label(self):
|
||||
actual = list(range(8))
|
||||
predicted = [0, 1, 2, 3, 4, 5, 6, 0]
|
||||
metrics = train.classification_metrics(actual, predicted)
|
||||
self.assertEqual(7 / 8, metrics["accuracy"])
|
||||
self.assertEqual(0.0, metrics["perPurposeRecall"]["writing"])
|
||||
self.assertEqual(1.0, metrics["perPurposeRecall"]["planning"])
|
||||
|
||||
def test_thresholds_preserve_confidence_nesting(self):
|
||||
probabilities = [0.99, 0.95, 0.85, 0.75, 0.65]
|
||||
margins = [0.95, 0.85, 0.60, 0.40, 0.20]
|
||||
correct = [True, True, True, False, False]
|
||||
thresholds = train.choose_confidence_thresholds(
|
||||
probabilities,
|
||||
margins,
|
||||
correct,
|
||||
high_precision=1.0,
|
||||
accepted_precision=0.75,
|
||||
)
|
||||
self.assertGreaterEqual(
|
||||
thresholds["high"]["minimumScore"],
|
||||
thresholds["medium"]["minimumScore"],
|
||||
)
|
||||
self.assertGreater(thresholds["medium"]["validationAcceptedCoverage"], 0)
|
||||
|
||||
def test_expected_calibration_error_is_zero_for_perfect_extremes(self):
|
||||
self.assertEqual(
|
||||
0.0,
|
||||
train.expected_calibration_error([1.0, 0.0], [True, False]),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user