cesnimda
185 lines
11 KiB
Python
185 lines
11 KiB
Python
"""Supplement the immutable original factual oracle with recovery/association/provenance/abstention."""
|
|
import argparse
|
|
from collections import Counter
|
|
import importlib.util
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
|
|
from generate import fragments, load_corpus
|
|
|
|
|
|
BASELINE_PATH = Path(__file__).resolve().parents[2] / "cv-import/benchmarks/evaluate.py"
|
|
spec = importlib.util.spec_from_file_location("original_cv_import_oracle", BASELINE_PATH)
|
|
baseline = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(baseline)
|
|
RECORD_CATEGORIES = ("jobs", "education", "projects", "certifications", "languages")
|
|
|
|
|
|
def score(truth, predicted):
|
|
tp = (truth & predicted).total()
|
|
precision = baseline.ratio(tp, predicted.total())
|
|
recall = baseline.ratio(tp, truth.total())
|
|
return {"truePositive": tp, "falsePositive": (predicted - truth).total(),
|
|
"falseNegative": (truth - predicted).total(), "expected": truth.total(), "predicted": predicted.total(),
|
|
"precision": precision, "recall": recall,
|
|
"f1": 2 * precision * recall / (precision + recall) if precision + recall else 0.0}
|
|
|
|
|
|
def canonical(profile):
|
|
# Unresolved source is measured independently, so correct abstention is not a fabricated typed fact.
|
|
return {key: value for key, value in profile.items() if key != "otherSections"}
|
|
|
|
|
|
def unresolved_facts(groups):
|
|
result = Counter()
|
|
for group in groups:
|
|
title = baseline.normalise(group.get("title") or "")
|
|
for item in group.get("items", []):
|
|
# A PDF has explicit visual list markers; the DOCX text item omits its list marker.
|
|
result[(title, re.sub(r"^[-*•]\s+", "", baseline.normalise(item)))] += 1
|
|
return result
|
|
|
|
|
|
def record_scores(expected, actual):
|
|
result = {}
|
|
for category in RECORD_CATEGORIES:
|
|
truth, predicted = expected.get(category, []), actual.get(category, [])
|
|
available, exact_available = list(predicted), list(predicted)
|
|
complete = exact = 0
|
|
for item in truth:
|
|
facts = baseline.facts({category: [item]})
|
|
match = next((i for i, candidate in enumerate(available)
|
|
if not facts - baseline.facts({category: [candidate]})), None)
|
|
if match is not None:
|
|
complete += 1
|
|
available.pop(match)
|
|
match = next((i for i, candidate in enumerate(exact_available)
|
|
if facts == baseline.facts({category: [candidate]})), None)
|
|
if match is not None:
|
|
exact += 1
|
|
exact_available.pop(match)
|
|
result[category] = {"expected": len(truth), "predicted": len(predicted), "complete": complete, "exact": exact,
|
|
"completenessRecall": baseline.ratio(complete, len(truth)),
|
|
"exactPrecision": baseline.ratio(exact, len(predicted))}
|
|
return result
|
|
|
|
|
|
def provenance_score(case, profile, extraction):
|
|
checks = []
|
|
metadata = profile.get("metadata", {}).get("fields", {})
|
|
for path, wanted in case.get("provenance", {}).items():
|
|
matches = [block for block in extraction.get("blocks", [])
|
|
if baseline.normalise(wanted["text"]) in baseline.normalise(block["text"])]
|
|
occurrence = wanted.get("occurrence", 1)
|
|
target = matches[occurrence - 1] if len(matches) >= occurrence else None
|
|
observed = metadata.get(path, {})
|
|
correct = target is not None and observed.get("sourceBlockId") == target["id"] and observed.get("sourcePage") == target.get("page")
|
|
checks.append({"path": path, "sourceText": wanted["text"], "occurrence": occurrence,
|
|
"expectedBlockId": target["id"] if target else None,
|
|
"actualBlockId": observed.get("sourceBlockId"), "correct": correct,
|
|
"sourceOccurrenceRecovered": target is not None})
|
|
return {"expected": len(checks), "correct": sum(check["correct"] for check in checks),
|
|
"accuracy": baseline.ratio(sum(check["correct"] for check in checks), len(checks)), "checks": checks}
|
|
|
|
|
|
def evaluate_case(case, output, extraction):
|
|
expected, actual = canonical(case["expected"]), canonical(output.get("profile", {}))
|
|
truth, predicted = baseline.facts(expected), baseline.facts(actual)
|
|
factual = baseline.evaluate_case(expected, actual)
|
|
factual.update(score(truth, predicted))
|
|
field_names = {re.sub(r"\[[^\]]*\]", "", key[0]) for key in truth | predicted}
|
|
per_field = {name: score(Counter({k: n for k, n in truth.items() if re.sub(r"\[[^\]]*\]", "", k[0]) == name}),
|
|
Counter({k: n for k, n in predicted.items() if re.sub(r"\[[^\]]*\]", "", k[0]) == name}))
|
|
for name in sorted(field_names)}
|
|
unresolved = score(unresolved_facts(case.get("unresolved", [])),
|
|
unresolved_facts(output.get("profile", {}).get("otherSections", [])))
|
|
wanted_fragments = Counter(baseline.normalise(text) for text in fragments(case))
|
|
extracted_text = baseline.normalise(extraction.get("text", ""))
|
|
recovered = sum(min(count, extracted_text.count(text)) for text, count in wanted_fragments.items())
|
|
required = case.get("abstentionRequired", False)
|
|
records = record_scores(expected, actual)
|
|
return {"id": case.get("id"), "split": case.get("split"), "completed": "profile" in output,
|
|
"factual": factual, "perField": per_field, "records": records,
|
|
"unresolved": unresolved,
|
|
"abstention": {"required": required, "correct": required and factual["falsePositive"] == 0
|
|
and factual["falseNegative"] == 0 and unresolved["falseNegative"] == 0 and unresolved["falsePositive"] == 0
|
|
and all(record["expected"] == record["predicted"] == record["exact"] for record in records.values())},
|
|
"provenance": provenance_score(case, output.get("profile", {}), extraction),
|
|
"doclingRecovery": {"completed": extraction.get("ok") is True, "expectedFragments": wanted_fragments.total(),
|
|
"recoveredFragments": recovered, "literalFragmentRecall": baseline.ratio(recovered, wanted_fragments.total()),
|
|
"method": extraction.get("method", "NOT RUN")},
|
|
"interpretationTiming": output.get("timing", {}), "reviewRequired": output.get("reviewRequired", "NOT MEASURED")}
|
|
|
|
|
|
def aggregate_scores(items):
|
|
tp = sum(item["truePositive"] for item in items)
|
|
fp = sum(item["falsePositive"] for item in items)
|
|
fn = sum(item["falseNegative"] for item in items)
|
|
precision, recall = baseline.ratio(tp, tp + fp), baseline.ratio(tp, tp + fn)
|
|
return {"truePositive": tp, "falsePositive": fp, "falseNegative": fn, "expected": tp + fn, "predicted": tp + fp,
|
|
"precision": precision, "recall": recall,
|
|
"f1": 2 * precision * recall / (precision + recall) if precision + recall else 0.0}
|
|
|
|
|
|
def evaluate_run(corpus, outputs, extractions):
|
|
ids = [case["id"] for case in corpus]
|
|
if len(ids) != len(set(ids)) or set(outputs) - set(ids) or set(extractions) - set(ids):
|
|
raise ValueError("Duplicate annotations or unexpected output/extraction IDs.")
|
|
cases = [evaluate_case(case, outputs.get(case["id"], {}), extractions.get(case["id"], {})) for case in corpus]
|
|
field_names = sorted({field for case in cases for field in case["perField"]})
|
|
recovered = sum(case["doclingRecovery"]["recoveredFragments"] for case in cases)
|
|
fragments_count = sum(case["doclingRecovery"]["expectedFragments"] for case in cases)
|
|
provenance_expected = sum(case["provenance"]["expected"] for case in cases)
|
|
provenance_correct = sum(case["provenance"]["correct"] for case in cases)
|
|
records = {}
|
|
for category in RECORD_CATEGORIES:
|
|
record = {key: sum(case["records"][category][key] for case in cases) for key in ("expected", "predicted", "complete", "exact")}
|
|
record.update(completenessRecall=baseline.ratio(record["complete"], record["expected"]), exactPrecision=baseline.ratio(record["exact"], record["predicted"]))
|
|
records[category] = record
|
|
abstentions = [case for case in cases if case["abstention"]["required"]]
|
|
return {"caseCount": len(cases), "completedCases": sum(case["completed"] for case in cases),
|
|
"completion": baseline.ratio(sum(case["completed"] for case in cases), len(cases)),
|
|
"doclingRecovery": {"completedCases": sum(case["doclingRecovery"]["completed"] for case in cases),
|
|
"expectedFragments": fragments_count, "recoveredFragments": recovered,
|
|
"literalFragmentRecall": baseline.ratio(recovered, fragments_count)},
|
|
"interpretation": aggregate_scores([case["factual"] for case in cases]),
|
|
"perField": {name: aggregate_scores([case["perField"][name] for case in cases if name in case["perField"]]) for name in field_names},
|
|
"records": records, "unresolvedSource": aggregate_scores([case["unresolved"] for case in cases]),
|
|
"provenance": {"expected": provenance_expected, "correct": provenance_correct, "accuracy": baseline.ratio(provenance_correct, provenance_expected)},
|
|
"abstention": {"requiredCases": len(abstentions), "correctCases": sum(case["abstention"]["correct"] for case in abstentions),
|
|
"accuracy": baseline.ratio(sum(case["abstention"]["correct"] for case in abstentions), len(abstentions))},
|
|
"reviewedMapping": "NOT RUN", "realAiAccuracy": "BLOCKED", "cases": cases,
|
|
"limits": "Synthetic cases only; no population accuracy estimate. Text recovery does not prove reading order, table interpretation or record associations. Typed facts exclude unresolved source, which is scored separately. Interpreter outputs do not qualify user mappings, reviewed persistence, human acceptance or real model accuracy."}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--corpus", type=Path, default=Path(__file__).with_name("corpus.json"))
|
|
parser.add_argument("--actual", type=Path, required=True, help="Original harness profile outputs keyed by logical ID")
|
|
parser.add_argument("--extractions", type=Path, required=True)
|
|
parser.add_argument("--format", choices=("pdf", "docx"), required=True)
|
|
parser.add_argument("--split", choices=("development", "frozen"))
|
|
parser.add_argument("--output", type=Path, required=True)
|
|
args = parser.parse_args()
|
|
corpus = load_corpus(args.corpus)
|
|
outputs = json.loads(args.actual.read_text(encoding="utf-8"))
|
|
if args.split:
|
|
corpus = [case for case in corpus if case["split"] == args.split]
|
|
outputs = {case["id"]: outputs[case["id"]] for case in corpus if case["id"] in outputs}
|
|
extractions = {}
|
|
for case in corpus:
|
|
path = args.extractions / f"{case['id']}.{args.format}.extracted.json"
|
|
if path.exists():
|
|
extractions[case["id"]] = json.loads(path.read_text(encoding="utf-8"))
|
|
result = evaluate_run(corpus, outputs, extractions)
|
|
result.update(format=args.format, split=args.split or "all")
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
with args.output.open("x", encoding="utf-8") as target:
|
|
json.dump(result, target, indent=2, ensure_ascii=False)
|
|
print(json.dumps({key: value for key, value in result.items() if key not in {"cases", "perField"}}, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|