cesnimda
111 lines
6.7 KiB
Python
111 lines
6.7 KiB
Python
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
|
|
from docx import Document
|
|
from pypdf import PdfReader
|
|
|
|
from generate import fragments, generate_case, load_corpus
|
|
from evaluate import evaluate_case, evaluate_run
|
|
|
|
|
|
class GeneratorTests(unittest.TestCase):
|
|
def test_corpus_is_unique_split_and_independently_annotated(self):
|
|
corpus = load_corpus(Path(__file__).with_name("corpus.json"))
|
|
self.assertGreaterEqual(len(corpus), 16)
|
|
self.assertEqual({c["split"] for c in corpus}, {"development", "frozen"})
|
|
ambiguous = [c for c in corpus if c.get("abstentionRequired")]
|
|
self.assertTrue(ambiguous)
|
|
self.assertTrue(all(c["unresolved"] for c in ambiguous))
|
|
|
|
def test_provenance_expectations_name_existing_literal_source_occurrences(self):
|
|
for case in load_corpus(Path(__file__).with_name("corpus.json")):
|
|
for path, reference in case.get("provenance", {}).items():
|
|
matches = [text for text in fragments(case) if reference["text"] in text]
|
|
self.assertGreaterEqual(len(matches), reference.get("occurrence", 1), (case["id"], path))
|
|
value = {**case["expected"], "otherSections": case.get("unresolved", [])}
|
|
for part in path.split("."):
|
|
value = value[int(part)] if isinstance(value, list) else value[part]
|
|
if isinstance(value, str):
|
|
self.assertIn(value, reference["text"], (case["id"], path))
|
|
|
|
def test_generator_creates_real_lists_multirow_table_unicode_and_page_break(self):
|
|
case = {"id": "structure", "blocks": [{"type": "heading", "text": "Utdanning"},
|
|
{"type": "list", "text": "Åse Øvrebø"},
|
|
{"type": "table", "headers": ["Institution", "Qualification"], "rows": [["A", "BSc"], ["B", "MSc"]]},
|
|
{"type": "page_break"}, {"text": "Tromsø"}]}
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
paths = generate_case(case, Path(directory))
|
|
word = Document(paths["docx"])
|
|
self.assertEqual(word.paragraphs[0].style.name, "Heading 2")
|
|
self.assertEqual(word.paragraphs[1].style.name, "List Bullet")
|
|
self.assertEqual(len(word.tables[0].rows), 3)
|
|
self.assertEqual([c.text for c in word.tables[0].rows[0].cells], ["Institution", "Qualification"])
|
|
pdf = PdfReader(paths["pdf"])
|
|
self.assertEqual(len(pdf.pages), 2)
|
|
self.assertIn("Åse Øvrebø", pdf.pages[0].extract_text())
|
|
self.assertIn("Tromsø", pdf.pages[1].extract_text())
|
|
with self.assertRaises(FileExistsError):
|
|
generate_case(case, Path(directory))
|
|
|
|
|
|
class OracleTests(unittest.TestCase):
|
|
def test_correct_abstention_is_scored_separately_from_typed_recall(self):
|
|
case = {"expected": {}, "blocks": [{"text": "A"}, {"text": "Engineer"}],
|
|
"unresolved": [{"title": None, "items": ["A", "Engineer"]}], "abstentionRequired": True}
|
|
result = evaluate_case(case, {"profile": {"otherSections": [{"items": ["A", "Engineer"]}]}},
|
|
{"ok": True, "text": "A\nEngineer", "blocks": []})
|
|
self.assertTrue(result["abstention"]["correct"])
|
|
self.assertEqual(result["factual"]["falsePositive"], 0)
|
|
self.assertEqual(result["unresolved"]["recall"], 1)
|
|
fabricated = evaluate_case(case, {"profile": {"jobs": [{"company": "A", "title": "Engineer"}]}}, {})
|
|
self.assertFalse(fabricated["abstention"]["correct"])
|
|
self.assertEqual(fabricated["factual"]["falsePositive"], 2)
|
|
|
|
def test_second_occurrence_provenance_cannot_point_to_first_matching_block(self):
|
|
case = {"expected": {"jobs": [{"company": "A", "title": "Engineer"}]}, "blocks": [],
|
|
"provenance": {"jobs.0.company": {"text": "Company: A", "occurrence": 2}}}
|
|
profile = {"jobs": case["expected"]["jobs"], "metadata": {"fields": {
|
|
"jobs.0.company": {"sourceBlockId": "b1", "sourcePage": 1}}}}
|
|
extraction = {"blocks": [{"id": "b1", "text": "Company: A", "page": 1},
|
|
{"id": "b2", "text": "Company: A", "page": 2}]}
|
|
result = evaluate_case(case, {"profile": profile}, extraction)
|
|
self.assertEqual(result["factual"]["recall"], 1)
|
|
self.assertEqual(result["provenance"]["correct"], 0)
|
|
profile["metadata"]["fields"]["jobs.0.company"] = {"sourceBlockId": "b2", "sourcePage": 2}
|
|
self.assertEqual(evaluate_case(case, {"profile": profile}, extraction)["provenance"]["correct"], 1)
|
|
|
|
def test_swapped_associations_fail_records_and_dates_without_hiding_extra_fields(self):
|
|
truth = {"jobs": [{"company": "A", "title": "Engineer", "start": "2020", "end": "2021", "bullets": ["First"]},
|
|
{"company": "A", "title": "Engineer", "start": "2022", "end": "2023", "bullets": ["Second"]}]}
|
|
actual = [{**truth["jobs"][0], "bullets": ["Second"]}, {**truth["jobs"][1], "bullets": ["First"]}]
|
|
result = evaluate_case({"expected": truth, "blocks": []}, {"profile": {"jobs": actual}}, {})
|
|
self.assertEqual(result["records"]["jobs"]["complete"], 0)
|
|
self.assertEqual(result["factual"]["falsePositive"], 2)
|
|
self.assertLess(result["perField"]["jobs.bullets"]["f1"], 1)
|
|
|
|
def test_missing_output_counts_against_completion_and_recovery(self):
|
|
corpus = [{"id": "a", "split": "frozen", "expected": {"skills": ["C#"]}, "blocks": [{"text": "C#"}]}]
|
|
result = evaluate_run(corpus, {}, {})
|
|
self.assertEqual(result["completion"], 0)
|
|
self.assertEqual(result["interpretation"]["recall"], 0)
|
|
self.assertEqual(result["doclingRecovery"]["literalFragmentRecall"], 0)
|
|
self.assertEqual(result["reviewedMapping"], "NOT RUN")
|
|
self.assertEqual(result["realAiAccuracy"], "BLOCKED")
|
|
|
|
def test_unknown_source_and_list_markers_have_independent_preservation_score(self):
|
|
case = {"expected": {}, "blocks": [], "unresolved": [{"title": "Work Experience", "items": ["Orphan detail"]}]}
|
|
result = evaluate_case(case, {"profile": {"otherSections": [{"title": "Work Experience", "items": ["- Orphan detail", "Unexpected"]}]}}, {})
|
|
self.assertEqual(result["unresolved"]["truePositive"], 1)
|
|
self.assertEqual(result["unresolved"]["falsePositive"], 1)
|
|
self.assertEqual(result["unresolved"]["recall"], 1)
|
|
|
|
def test_an_empty_typed_record_is_not_correct_abstention(self):
|
|
case = {"expected": {}, "blocks": [], "unresolved": [{"items": ["Unknown"]}], "abstentionRequired": True}
|
|
actual = {"jobs": [{}], "otherSections": [{"items": ["Unknown"]}]}
|
|
self.assertFalse(evaluate_case(case, {"profile": actual}, {})["abstention"]["correct"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|