Merge nucleic/lucid-north-quail-rnvt into dev

tests/test_train.py

@@ -0,0 +1,46 @@
+import sys
+import unittest
+from pathlib import Path
+
+
+MODULE_DIR = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(MODULE_DIR))
+
+import train
+
+
+class MetricsTests(unittest.TestCase):
+    def test_classification_metrics_include_every_label(self):
+        actual = list(range(8))
+        predicted = [0, 1, 2, 3, 4, 5, 6, 0]
+        metrics = train.classification_metrics(actual, predicted)
+        self.assertEqual(7 / 8, metrics["accuracy"])
+        self.assertEqual(0.0, metrics["perPurposeRecall"]["writing"])
+        self.assertEqual(1.0, metrics["perPurposeRecall"]["planning"])
+
+    def test_thresholds_preserve_confidence_nesting(self):
+        probabilities = [0.99, 0.95, 0.85, 0.75, 0.65]
+        margins = [0.95, 0.85, 0.60, 0.40, 0.20]
+        correct = [True, True, True, False, False]
+        thresholds = train.choose_confidence_thresholds(
+            probabilities,
+            margins,
+            correct,
+            high_precision=1.0,
+            accepted_precision=0.75,
+        )
+        self.assertGreaterEqual(
+            thresholds["high"]["minimumScore"],
+            thresholds["medium"]["minimumScore"],
+        )
+        self.assertGreater(thresholds["medium"]["validationAcceptedCoverage"], 0)
+
+    def test_expected_calibration_error_is_zero_for_perfect_extremes(self):
+        self.assertEqual(
+            0.0,
+            train.expected_calibration_error([1.0, 0.0], [True, False]),
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()