cesnimda
107 lines
5.5 KiB
Python
107 lines
5.5 KiB
Python
"""Score exhaustive synthetic Career Profile annotations; never substitutes for model qualification."""
|
|
import argparse
|
|
from collections import Counter
|
|
import json
|
|
from pathlib import Path
|
|
import unicodedata
|
|
|
|
|
|
def normalise(value):
|
|
return " ".join(unicodedata.normalize("NFC", str(value)).split()).casefold()
|
|
|
|
|
|
def facts(profile):
|
|
"""Keep record associations and repeated claims, ignoring implementation-only metadata."""
|
|
result = Counter()
|
|
|
|
def walk(value, path):
|
|
if isinstance(value, dict):
|
|
for key, item in value.items():
|
|
if key not in {"metadata", "version", "sections", "id", "startDate", "endDate", "dateNormalized"}:
|
|
walk(item, f"{path}.{key}" if path else key)
|
|
elif isinstance(value, list):
|
|
for item in value:
|
|
record = ""
|
|
if isinstance(item, dict):
|
|
identity = [key for key in ("company", "title", "institution", "qualification", "name", "start", "end", "date") if item.get(key)]
|
|
record = "[" + "|".join(f"{key}={normalise(item[key])}" for key in identity) + "]"
|
|
walk(item, path + record)
|
|
elif value not in (None, "", False):
|
|
result[(path, normalise(value))] += 1
|
|
|
|
walk(profile, "")
|
|
return result
|
|
|
|
|
|
def ratio(numerator, denominator):
|
|
return numerator / denominator if denominator else 1.0
|
|
|
|
|
|
def evaluate_case(expected, actual):
|
|
truth, predicted = facts(expected), facts(actual)
|
|
correct = sum((truth & predicted).values())
|
|
date_truth = Counter({key: count for key, count in truth.items() if key[0].rsplit(".", 1)[-1] in {"start", "end", "date", "isCurrent"}})
|
|
date_predicted = Counter({key: count for key, count in predicted.items() if key[0].rsplit(".", 1)[-1] in {"start", "end", "date", "isCurrent"}})
|
|
category_equal = lambda category: facts({category: expected.get(category, [])}) == facts({category: actual.get(category, [])})
|
|
return {
|
|
"truePositive": correct,
|
|
"falsePositive": sum((predicted - truth).values()),
|
|
"falseNegative": sum((truth - predicted).values()),
|
|
"precision": ratio(correct, predicted.total()),
|
|
"recall": ratio(correct, truth.total()),
|
|
"datePrecision": ratio((date_truth & date_predicted).total(), date_predicted.total()),
|
|
"dateRecall": ratio((date_truth & date_predicted).total(), date_truth.total()),
|
|
"employmentExact": category_equal("jobs"),
|
|
"educationExact": category_equal("education"),
|
|
"skillsExact": category_equal("skills"),
|
|
"unsupportedClaims": [{"path": path, "value": value, "count": count} for (path, value), count in (predicted - truth).items()],
|
|
"missingClaims": [{"path": path, "value": value, "count": count} for (path, value), count in (truth - predicted).items()],
|
|
}
|
|
|
|
|
|
def evaluate_run(corpus, outputs):
|
|
ids = [case["id"] for case in corpus]
|
|
if len(set(ids)) != len(ids) or set(outputs) - set(ids):
|
|
raise ValueError("Duplicate annotations or unexpected output cases.")
|
|
cases = []
|
|
for case in corpus:
|
|
output = outputs.get(case["id"], {})
|
|
result = evaluate_case(case["expected"], output.get("profile", {}))
|
|
result.update(id=case["id"], completed="profile" in output, split=case.get("split", "development"),
|
|
extractionMethod=output.get("extractionMethod", "NOT RUN"),
|
|
timing=output.get("timing", {}), reviewRequired=output.get("reviewRequired", "NOT MEASURED"))
|
|
cases.append(result)
|
|
tp = sum(case["truePositive"] for case in cases)
|
|
fp = sum(case["falsePositive"] for case in cases)
|
|
fn = sum(case["falseNegative"] for case in cases)
|
|
return {"cases": cases, "caseCount": len(cases), "completion": ratio(sum(c["completed"] for c in cases), len(cases)),
|
|
"microPrecision": ratio(tp, tp + fp), "microRecall": ratio(tp, tp + fn),
|
|
"unsupportedClaimCount": fp, "missingClaimCount": fn,
|
|
"employmentExactCases": sum(c["employmentExact"] and c["completed"] for c in cases),
|
|
"educationExactCases": sum(c["educationExact"] and c["completed"] for c in cases),
|
|
"realAiAccuracy": "BLOCKED", "mockedAiCountsAsModelAccuracy": False,
|
|
"limits": "Synthetic exhaustive annotations only. Unsupported claims include association errors, not necessarily invented text. No population accuracy estimate, human acceptance or model inference is implied."}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--corpus", type=Path, default=Path(__file__).with_name("corpus.json"))
|
|
parser.add_argument("--actual", required=True, type=Path, help="JSON object keyed by case ID, each with a canonical profile.")
|
|
parser.add_argument("--output", required=True, type=Path)
|
|
parser.add_argument("--split", choices=("development", "frozen"))
|
|
args = parser.parse_args()
|
|
corpus = json.loads(args.corpus.read_text(encoding="utf-8"))
|
|
outputs = json.loads(args.actual.read_text(encoding="utf-8"))
|
|
if args.split:
|
|
corpus = [case for case in corpus if case["split"] == args.split]
|
|
outputs = {case["id"]: outputs[case["id"]] for case in corpus if case["id"] in outputs}
|
|
report = evaluate_run(corpus, outputs)
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
with args.output.open("x", encoding="utf-8") as target:
|
|
json.dump(report, target, indent=2, ensure_ascii=False)
|
|
print(json.dumps({key: value for key, value in report.items() if key != "cases"}, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|