Merge nucleic/sleek-ember-seal-uady into dev

tests/test_eval.py

@@ -23,5 +23,33 @@ class TierDriftTests(unittest.TestCase):
         self.assertLessEqual(report["maximumTierDrift"], 1)
 
 
+class PredictionAgreementTests(unittest.TestCase):
+    def test_reports_accuracy_transitions_and_scorable_agreement(self):
+        records = [
+            {"prompt": "one", "slice": "core"},
+            {"prompt": "two", "slice": "boundary"},
+            {"prompt": "three", "slice": "vague-eval"},
+            {"prompt": "four", "slice": "core"},
+        ]
+        report = purpose_eval.prediction_agreement(
+            records,
+            actual=[0, 1, 2, 3],
+            reference=[0, 0, 3, 4],
+            candidate=[1, 1, 4, 5],
+        )
+        self.assertEqual(0.0, report["labelAgreement"])
+        self.assertEqual(0.0, report["scoredLabelAgreement"])
+        self.assertEqual(
+            {
+                "correctToIncorrect": 1,
+                "differentIncorrectLabel": 2,
+                "incorrectToCorrect": 1,
+            },
+            report["transitionCounts"],
+        )
+        self.assertEqual(2, report["bySlice"]["core"]["records"])
+        self.assertEqual(4, len(report["disagreements"]))
+
+
 if __name__ == "__main__":
     unittest.main()

tests/test_export.py

@@ -0,0 +1,51 @@
+import sys
+import unittest
+from collections import Counter
+from pathlib import Path
+
+
+MODULE_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(MODULE_DIR))
+
+import export
+
+
+class CalibrationSampleTests(unittest.TestCase):
+    def test_sample_is_exact_deterministic_and_stratified(self):
+        records = []
+        for index in range(100):
+            records.append(
+                {
+                    "prompt": f"prompt {index}",
+                    "purpose": "planning" if index < 80 else "writing",
+                    "slice": "core" if index % 2 else "boundary",
+                    "lang": "en" if index % 5 else "fr",
+                }
+            )
+        first = export.stratified_calibration_sample(records, 25, seed=42)
+        second = export.stratified_calibration_sample(records, 25, seed=42)
+        self.assertEqual(25, len(first))
+        self.assertEqual(
+            [item["prompt"] for item in first],
+            [item["prompt"] for item in second],
+        )
+        purposes = Counter(item["purpose"] for item in first)
+        self.assertEqual({"planning": 20, "writing": 5}, dict(purposes))
+
+    def test_sample_caps_at_population(self):
+        records = [
+            {
+                "prompt": "one",
+                "purpose": "planning",
+                "slice": "core",
+                "lang": "en",
+            }
+        ]
+        self.assertEqual(
+            records,
+            export.stratified_calibration_sample(records, 10, seed=1),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_train.py

@@ -12,6 +12,16 @@ import train
 
 
 class MetricsTests(unittest.TestCase):
+    def test_boundary_training_weight_is_opt_in(self):
+        self.assertEqual(
+            2.0,
+            train.training_weight({"slice": "boundary"}, boundary_weight=2.0),
+        )
+        self.assertEqual(
+            1.0,
+            train.training_weight({"slice": "core"}, boundary_weight=2.0),
+        )
+
     def test_classification_metrics_include_every_label(self):
         actual = list(range(8))
         predicted = [0, 1, 2, 3, 4, 5, 6, 0]