47 lines
1.5 KiB
Python
47 lines
1.5 KiB
Python
|
import sys
|
||
|
|
import unittest
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
|
||
|
|
MODULE_DIR = Path(__file__).resolve().parents[1]
|
||
|
|
sys.path.insert(0, str(MODULE_DIR))
|
||
|
|
|
||
|
|
import train
|
||
|
|
|
||
|
|
|
||
|
|
class MetricsTests(unittest.TestCase):
|
||
|
|
def test_classification_metrics_include_every_label(self):
|
||
|
|
actual = list(range(8))
|
||
|
|
predicted = [0, 1, 2, 3, 4, 5, 6, 0]
|
||
|
|
metrics = train.classification_metrics(actual, predicted)
|
||
|
|
self.assertEqual(7 / 8, metrics["accuracy"])
|
||
|
|
self.assertEqual(0.0, metrics["perPurposeRecall"]["writing"])
|
||
|
|
self.assertEqual(1.0, metrics["perPurposeRecall"]["planning"])
|
||
|
|
|
||
|
|
def test_thresholds_preserve_confidence_nesting(self):
|
||
|
|
probabilities = [0.99, 0.95, 0.85, 0.75, 0.65]
|
||
|
|
margins = [0.95, 0.85, 0.60, 0.40, 0.20]
|
||
|
|
correct = [True, True, True, False, False]
|
||
|
|
thresholds = train.choose_confidence_thresholds(
|
||
|
|
probabilities,
|
||
|
|
margins,
|
||
|
|
correct,
|
||
|
|
high_precision=1.0,
|
||
|
|
accepted_precision=0.75,
|
||
|
|
)
|
||
|
|
self.assertGreaterEqual(
|
||
|
|
thresholds["high"]["minimumScore"],
|
||
|
|
thresholds["medium"]["minimumScore"],
|
||
|
|
)
|
||
|
|
self.assertGreater(thresholds["medium"]["validationAcceptedCoverage"], 0)
|
||
|
|
|
||
|
|
def test_expected_calibration_error_is_zero_for_perfect_extremes(self):
|
||
|
|
self.assertEqual(
|
||
|
|
0.0,
|
||
|
|
train.expected_calibration_error([1.0, 0.0], [True, False]),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
unittest.main()
|