test(cv-import): add frozen corpus and bounded accuracy qualification

2026-10-26 16:00:55 +01:00
parent cd67c0df21
commit b62962c68a
12 changed files with 541 additions and 1 deletions
@@ -0,0 +1,106 @@
+"""Score exhaustive synthetic Career Profile annotations; never substitutes for model qualification."""
+import argparse
+from collections import Counter
+import json
+from pathlib import Path
+import unicodedata
+
+
+def normalise(value):
+    return " ".join(unicodedata.normalize("NFC", str(value)).split()).casefold()
+
+
+def facts(profile):
+    """Keep record associations and repeated claims, ignoring implementation-only metadata."""
+    result = Counter()
+
+    def walk(value, path):
+        if isinstance(value, dict):
+            for key, item in value.items():
+                if key not in {"metadata", "version", "sections", "id", "startDate", "endDate", "dateNormalized"}:
+                    walk(item, f"{path}.{key}" if path else key)
+        elif isinstance(value, list):
+            for item in value:
+                record = ""
+                if isinstance(item, dict):
+                    identity = [key for key in ("company", "title", "institution", "qualification", "name", "start", "end", "date") if item.get(key)]
+                    record = "[" + "|".join(f"{key}={normalise(item[key])}" for key in identity) + "]"
+                walk(item, path + record)
+        elif value not in (None, "", False):
+            result[(path, normalise(value))] += 1
+
+    walk(profile, "")
+    return result
+
+
+def ratio(numerator, denominator):
+    return numerator / denominator if denominator else 1.0
+
+
+def evaluate_case(expected, actual):
+    truth, predicted = facts(expected), facts(actual)
+    correct = sum((truth & predicted).values())
+    date_truth = Counter({key: count for key, count in truth.items() if key[0].rsplit(".", 1)[-1] in {"start", "end", "date", "isCurrent"}})
+    date_predicted = Counter({key: count for key, count in predicted.items() if key[0].rsplit(".", 1)[-1] in {"start", "end", "date", "isCurrent"}})
+    category_equal = lambda category: facts({category: expected.get(category, [])}) == facts({category: actual.get(category, [])})
+    return {
+        "truePositive": correct,
+        "falsePositive": sum((predicted - truth).values()),
+        "falseNegative": sum((truth - predicted).values()),
+        "precision": ratio(correct, predicted.total()),
+        "recall": ratio(correct, truth.total()),
+        "datePrecision": ratio((date_truth & date_predicted).total(), date_predicted.total()),
+        "dateRecall": ratio((date_truth & date_predicted).total(), date_truth.total()),
+        "employmentExact": category_equal("jobs"),
+        "educationExact": category_equal("education"),
+        "skillsExact": category_equal("skills"),
+        "unsupportedClaims": [{"path": path, "value": value, "count": count} for (path, value), count in (predicted - truth).items()],
+        "missingClaims": [{"path": path, "value": value, "count": count} for (path, value), count in (truth - predicted).items()],
+    }
+
+
+def evaluate_run(corpus, outputs):
+    ids = [case["id"] for case in corpus]
+    if len(set(ids)) != len(ids) or set(outputs) - set(ids):
+        raise ValueError("Duplicate annotations or unexpected output cases.")
+    cases = []
+    for case in corpus:
+        output = outputs.get(case["id"], {})
+        result = evaluate_case(case["expected"], output.get("profile", {}))
+        result.update(id=case["id"], completed="profile" in output, split=case.get("split", "development"),
+                      extractionMethod=output.get("extractionMethod", "NOT RUN"),
+                      timing=output.get("timing", {}), reviewRequired=output.get("reviewRequired", "NOT MEASURED"))
+        cases.append(result)
+    tp = sum(case["truePositive"] for case in cases)
+    fp = sum(case["falsePositive"] for case in cases)
+    fn = sum(case["falseNegative"] for case in cases)
+    return {"cases": cases, "caseCount": len(cases), "completion": ratio(sum(c["completed"] for c in cases), len(cases)),
+            "microPrecision": ratio(tp, tp + fp), "microRecall": ratio(tp, tp + fn),
+            "unsupportedClaimCount": fp, "missingClaimCount": fn,
+            "employmentExactCases": sum(c["employmentExact"] and c["completed"] for c in cases),
+            "educationExactCases": sum(c["educationExact"] and c["completed"] for c in cases),
+            "realAiAccuracy": "BLOCKED", "mockedAiCountsAsModelAccuracy": False,
+            "limits": "Synthetic exhaustive annotations only. Unsupported claims include association errors, not necessarily invented text. No population accuracy estimate, human acceptance or model inference is implied."}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--corpus", type=Path, default=Path(__file__).with_name("corpus.json"))
+    parser.add_argument("--actual", required=True, type=Path, help="JSON object keyed by case ID, each with a canonical profile.")
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument("--split", choices=("development", "frozen"))
+    args = parser.parse_args()
+    corpus = json.loads(args.corpus.read_text(encoding="utf-8"))
+    outputs = json.loads(args.actual.read_text(encoding="utf-8"))
+    if args.split:
+        corpus = [case for case in corpus if case["split"] == args.split]
+        outputs = {case["id"]: outputs[case["id"]] for case in corpus if case["id"] in outputs}
+    report = evaluate_run(corpus, outputs)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    with args.output.open("x", encoding="utf-8") as target:
+        json.dump(report, target, indent=2, ensure_ascii=False)
+    print(json.dumps({key: value for key, value in report.items() if key != "cases"}, indent=2))
+
+
+if __name__ == "__main__":
+    main()