cesnimda
50 lines
2.7 KiB
Python
50 lines
2.7 KiB
Python
import unittest
|
|
|
|
from evaluate import evaluate_case, evaluate_run
|
|
|
|
|
|
class EvaluationTests(unittest.TestCase):
|
|
def test_repeated_employer_date_ranges_keep_their_own_achievements(self):
|
|
jobs = [{"company": "A", "title": "Engineer", "start": "2020", "end": "2021", "bullets": ["First role"]}, {"company": "A", "title": "Engineer", "start": "2022", "end": "2023", "bullets": ["Second role"]}]
|
|
swapped = [{**jobs[0], "bullets": jobs[1]["bullets"]}, {**jobs[1], "bullets": jobs[0]["bullets"]}]
|
|
result = evaluate_case({"jobs": jobs}, {"jobs": swapped})
|
|
self.assertFalse(result["employmentExact"])
|
|
self.assertLess(result["recall"], 1)
|
|
def test_extra_employment_is_an_unsupported_claim(self):
|
|
truth = {"jobs": [{"company": "Example AS", "title": "Engineer", "start": "2020"}]}
|
|
actual = {"jobs": truth["jobs"] + [{"company": "Invented AS", "title": "Director"}]}
|
|
result = evaluate_case(truth, actual)
|
|
self.assertEqual(result["falsePositive"], 2)
|
|
self.assertEqual(result["falseNegative"], 0)
|
|
self.assertEqual(result["employmentExact"], False)
|
|
|
|
def test_year_to_month_is_not_an_acceptable_normalisation(self):
|
|
result = evaluate_case({"jobs": [{"company": "A", "start": "2020"}]},
|
|
{"jobs": [{"company": "A", "start": "2020-01"}]})
|
|
self.assertLess(result["dateRecall"], 1)
|
|
|
|
def test_reordered_records_keep_associations(self):
|
|
jobs = [{"company": "A", "title": "Developer"}, {"company": "B", "title": "Designer"}]
|
|
self.assertEqual(evaluate_case({"jobs": jobs}, {"jobs": jobs[::-1]})["recall"], 1)
|
|
swapped = [{"company": "A", "title": "Designer"}, {"company": "B", "title": "Developer"}]
|
|
self.assertLess(evaluate_case({"jobs": jobs}, {"jobs": swapped})["recall"], 1)
|
|
|
|
def test_unicode_whitespace_only_normalisation_preserves_technical_skills(self):
|
|
result = evaluate_case({"skills": ["C++", "C#", " Blåbær "]},
|
|
{"skills": ["c++", "c#", "Blåbær"]})
|
|
self.assertEqual(result["precision"], 1)
|
|
self.assertEqual(result["recall"], 1)
|
|
self.assertLess(evaluate_case({"skills": ["C++"]}, {"skills": ["C"]})["precision"], 1)
|
|
|
|
def test_missing_case_counts_against_completion(self):
|
|
corpus = [{"id": "a", "expected": {"skills": ["Python"]}},
|
|
{"id": "b", "expected": {"skills": ["C#"]}}]
|
|
result = evaluate_run(corpus, {"a": {"profile": {"skills": ["Python"]}}})
|
|
self.assertEqual(result["completion"], .5)
|
|
self.assertEqual(result["microRecall"], .5)
|
|
self.assertEqual(result["realAiAccuracy"], "BLOCKED")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|