test(cv-import): add frozen corpus and bounded accuracy qualification

2026-10-26 16:00:55 +01:00
parent cd67c0df21
commit b62962c68a
12 changed files with 541 additions and 1 deletions
@@ -0,0 +1,120 @@
+"""Generate deterministic, nonpersonal PDF/DOCX benchmark documents and source hashes."""
+import argparse
+from hashlib import sha256
+from io import BytesIO
+import json
+from pathlib import Path
+import re
+
+from docx import Document
+from docx.oxml.ns import qn
+from PIL import Image, ImageDraw, ImageFont
+from reportlab import __file__ as reportlab_file
+from reportlab.lib.utils import ImageReader
+from reportlab.pdfbase import pdfmetrics
+from reportlab.pdfbase.ttfonts import TTFont
+from reportlab.pdfgen import canvas
+
+FONT_PATH = Path(reportlab_file).parent / "fonts/Vera.ttf"
+HEADINGS = {"professional summary", "work experience", "skills", "education", "certifications", "projects", "languages", "interests", "courses", "awards", "publications", "organisations", "references", "volunteer experience", "notes", "utdanning", "språk", "arbeidserfaring"}
+
+
+def generate_case(case, output):
+    identifier = case["id"]
+    if not re.fullmatch(r"[a-z0-9-]+", identifier):
+        raise ValueError("Fixture IDs may contain lowercase letters, numbers and hyphens only.")
+    output.mkdir(parents=True, exist_ok=True)
+    pdf = output / (identifier + ".pdf")
+    docx = output / (identifier + ".docx")
+    if pdf.exists() or docx.exists():
+        raise FileExistsError("Preserve prior fixtures and select a fresh output directory.")
+    pdfmetrics.registerFont(TTFont("CvImportBenchmark", str(FONT_PATH)))
+    document = canvas.Canvas(str(pdf), pagesize=(595, 842), invariant=1)
+    document.setTitle("Synthetic CV import benchmark: " + identifier)
+    document.setAuthor("Jobjakt synthetic fixture")
+    document.setFont("CvImportBenchmark", 10)
+    layout = case.get("layout", "single")
+    lines = case["lines"]
+    if layout == "scan":
+        image = Image.new("RGB", (1000, 1400), "white")
+        draw = ImageDraw.Draw(image)
+        font = ImageFont.truetype(str(FONT_PATH), 22)
+        draw.multiline_text((60, 60), "\n".join(lines), font=font, fill="black", spacing=12)
+        buffer = BytesIO()
+        image.save(buffer, format="PNG")
+        document.drawImage(ImageReader(buffer), 20, 20, width=555, height=777)
+        document.save()
+        return {"pdf": str(pdf)}
+    x, y = 40, 800
+    split = max(1, len(lines) // 2)
+    for index, line in enumerate(lines):
+        if line == "PAGE_BREAK" or y < 45:
+            document.showPage()
+            document.setFont("CvImportBenchmark", 10)
+            x, y = 40, 800
+            if line == "PAGE_BREAK":
+                continue
+        if layout == "columns" and index == split:
+            x, y = 305, 800
+        if layout == "table" and "|" in line:
+            cells = [part.strip() for part in line.split("|")]
+            width = 515 / len(cells)
+            for column, cell in enumerate(cells):
+                document.rect(40 + column * width, y - 5, width, 21)
+                document.drawString(44 + column * width, y + 2, cell)
+        else:
+            # Long names deliberately exercise wrapping without changing the source value.
+            words = line.split()
+            width = 250 if layout == "columns" else 515
+            wrapped = ""
+            for word in words:
+                if wrapped and pdfmetrics.stringWidth(wrapped + " " + word, "CvImportBenchmark", 10) > width:
+                    document.drawString(x, y, wrapped)
+                    y -= 14
+                    wrapped = word
+                else:
+                    wrapped = (wrapped + " " + word).strip()
+            document.drawString(x, y, wrapped)
+        y -= 24
+    document.save()
+
+    word = Document()
+    if layout == "columns":
+        columns = word.sections[0]._sectPr.find(qn("w:cols"))
+        columns.set(qn("w:num"), "2")
+    for line in lines:
+        if line == "PAGE_BREAK":
+            word.add_page_break()
+        elif layout == "table" and "|" in line:
+            cells = [part.strip() for part in line.split("|")]
+            table = word.add_table(rows=1, cols=len(cells))
+            for cell, text in zip(table.rows[0].cells, cells):
+                cell.text = text
+        else:
+            paragraph = word.add_paragraph(line)
+            if line.casefold() in HEADINGS:
+                paragraph.style = word.styles["Heading 2"]
+    word.save(docx)
+    return {"pdf": str(pdf), "docx": str(docx)}
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--corpus", type=Path, default=Path(__file__).with_name("corpus.json"))
+    parser.add_argument("--output", required=True, type=Path)
+    args = parser.parse_args()
+    if args.output.exists():
+        raise FileExistsError("Select a fresh output directory.")
+    corpus = json.loads(args.corpus.read_text(encoding="utf-8"))
+    manifest = {"corpusSha256": sha256(args.corpus.read_bytes()).hexdigest(), "documents": []}
+    for case in corpus:
+        for format_name, path in generate_case(case, args.output).items():
+            file = Path(path)
+            manifest["documents"].append({"caseId": case["id"], "split": case["split"], "format": format_name,
+                "file": file.name, "sha256": sha256(file.read_bytes()).hexdigest(), "bytes": file.stat().st_size})
+    (args.output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps({"documents": len(manifest["documents"]), "manifest": str(args.output / "manifest.json")}))
+
+
+if __name__ == "__main__":
+    main()