test(cv-import): add frozen corpus and bounded accuracy qualification

2026-10-26 16:00:55 +01:00
parent cd67c0df21
commit b62962c68a
12 changed files with 541 additions and 1 deletions
@@ -0,0 +1,49 @@
+import unittest
+
+from evaluate import evaluate_case, evaluate_run
+
+
+class EvaluationTests(unittest.TestCase):
+    def test_repeated_employer_date_ranges_keep_their_own_achievements(self):
+        jobs = [{"company": "A", "title": "Engineer", "start": "2020", "end": "2021", "bullets": ["First role"]}, {"company": "A", "title": "Engineer", "start": "2022", "end": "2023", "bullets": ["Second role"]}]
+        swapped = [{**jobs[0], "bullets": jobs[1]["bullets"]}, {**jobs[1], "bullets": jobs[0]["bullets"]}]
+        result = evaluate_case({"jobs": jobs}, {"jobs": swapped})
+        self.assertFalse(result["employmentExact"])
+        self.assertLess(result["recall"], 1)
+    def test_extra_employment_is_an_unsupported_claim(self):
+        truth = {"jobs": [{"company": "Example AS", "title": "Engineer", "start": "2020"}]}
+        actual = {"jobs": truth["jobs"] + [{"company": "Invented AS", "title": "Director"}]}
+        result = evaluate_case(truth, actual)
+        self.assertEqual(result["falsePositive"], 2)
+        self.assertEqual(result["falseNegative"], 0)
+        self.assertEqual(result["employmentExact"], False)
+
+    def test_year_to_month_is_not_an_acceptable_normalisation(self):
+        result = evaluate_case({"jobs": [{"company": "A", "start": "2020"}]},
+                               {"jobs": [{"company": "A", "start": "2020-01"}]})
+        self.assertLess(result["dateRecall"], 1)
+
+    def test_reordered_records_keep_associations(self):
+        jobs = [{"company": "A", "title": "Developer"}, {"company": "B", "title": "Designer"}]
+        self.assertEqual(evaluate_case({"jobs": jobs}, {"jobs": jobs[::-1]})["recall"], 1)
+        swapped = [{"company": "A", "title": "Designer"}, {"company": "B", "title": "Developer"}]
+        self.assertLess(evaluate_case({"jobs": jobs}, {"jobs": swapped})["recall"], 1)
+
+    def test_unicode_whitespace_only_normalisation_preserves_technical_skills(self):
+        result = evaluate_case({"skills": ["C++", "C#", "  Blåbær "]},
+                               {"skills": ["c++", "c#", "Blåbær"]})
+        self.assertEqual(result["precision"], 1)
+        self.assertEqual(result["recall"], 1)
+        self.assertLess(evaluate_case({"skills": ["C++"]}, {"skills": ["C"]})["precision"], 1)
+
+    def test_missing_case_counts_against_completion(self):
+        corpus = [{"id": "a", "expected": {"skills": ["Python"]}},
+                  {"id": "b", "expected": {"skills": ["C#"]}}]
+        result = evaluate_run(corpus, {"a": {"profile": {"skills": ["Python"]}}})
+        self.assertEqual(result["completion"], .5)
+        self.assertEqual(result["microRecall"], .5)
+        self.assertEqual(result["realAiAccuracy"], "BLOCKED")
+
+
+if __name__ == "__main__":
+    unittest.main()