diff --git a/JobTrackerApi.Tests/CvExtractionCoverageTests.cs b/JobTrackerApi.Tests/CvExtractionCoverageTests.cs new file mode 100644 index 0000000..4ab6eb9 --- /dev/null +++ b/JobTrackerApi.Tests/CvExtractionCoverageTests.cs @@ -0,0 +1,77 @@ +using JobTrackerApi.Models; +using Xunit; + +namespace JobTrackerApi.Tests; + +// Phase 2.1-b — CV extraction coverage. The structured model and StructuredCvProfileJson.FromSections +// already map Projects, Certifications and Languages headings into the profile; the gap was upstream +// (the AI /cv/normalize prompt never emitted those headings, so the sections were dropped). These +// tests lock the C# side so that once the normalized markdown carries # Projects / # Certifications / +// # Languages, they reach the structured profile — and so a future change can't silently regress it. +// +// Content shapes mirror what the (fixed) normalizer produces for the benchmark CV +// (Connor Babbington): a Projects section, and languages stated as "Name: Level". +public sealed class CvExtractionCoverageTests +{ + private static StructuredCvSection Section(string name, string content) => + new() { Name = name, Content = content }; + + [Fact] + public void FromSections_maps_a_Projects_heading_into_structured_projects() + { + var profile = StructuredCvProfileJson.FromSections(new[] + { + Section("Projects", + "JobTrack\nFull-stack job-application tracker (React, ASP.NET Core, SQLite, Docker).\n\n" + + "InboxIntel\nGmail analytics and safe bulk-cleanup tool in .NET 8 with PostgreSQL."), + }); + + Assert.Equal(2, profile.Projects.Count); + Assert.Contains(profile.Projects, p => p.Name == "JobTrack"); + Assert.Contains(profile.Projects, p => p.Name == "InboxIntel"); + } + + [Fact] + public void FromSections_maps_a_Certifications_heading_into_structured_certifications() + { + var profile = StructuredCvProfileJson.FromSections(new[] + { + Section("Certifications", + "Extended Diploma NVQ Level 3 in ICT\n\nAZ-900 Azure Fundamentals"), + }); + + Assert.NotEmpty(profile.Certifications); + Assert.Contains(profile.Certifications, c => (c.Name ?? "").Contains("NVQ", System.StringComparison.OrdinalIgnoreCase)); + } + + [Fact] + public void FromSections_maps_a_Languages_heading_with_levels() + { + var profile = StructuredCvProfileJson.FromSections(new[] + { + Section("Languages", "English: Native\nNorwegian: B1"), + }); + + Assert.Equal(2, profile.Languages.Count); + Assert.Contains(profile.Languages, l => l.Name == "English" && (l.Level ?? "").Contains("Native")); + Assert.Contains(profile.Languages, l => l.Name == "Norwegian" && (l.Level ?? "").Contains("B1")); + } + + // The benchmark CV has all four rich sections; confirm they coexist without one clobbering another. + [Fact] + public void FromSections_populates_projects_certifications_and_languages_together() + { + var profile = StructuredCvProfileJson.FromSections(new[] + { + Section("Skills", "C#\n.NET\nDocker"), + Section("Projects", "JobTrack\nJob-application tracker."), + Section("Certifications", "NVQ Level 3 in ICT"), + Section("Languages", "English: Native\nNorwegian: B1"), + }); + + Assert.NotEmpty(profile.Skills); + Assert.Single(profile.Projects); + Assert.NotEmpty(profile.Certifications); + Assert.Equal(2, profile.Languages.Count); + } +} diff --git a/JobTrackerApi/Controllers/ProfileCvController.cs b/JobTrackerApi/Controllers/ProfileCvController.cs index febd897..6117260 100644 --- a/JobTrackerApi/Controllers/ProfileCvController.cs +++ b/JobTrackerApi/Controllers/ProfileCvController.cs @@ -2124,7 +2124,7 @@ public sealed class ProfileCvController : ControllerBase private static bool LooksLikeNormalizedMarkdownCv(string text) { if (string.IsNullOrWhiteSpace(text)) return false; - return Regex.IsMatch(text, @"(?im)^#\s+(Contact|Professional Summary|Work Experience|Education|Skills|Languages|Interests)\s*$"); + return Regex.IsMatch(text, @"(?im)^#\s+(Contact|Professional Summary|Work Experience|Education|Skills|Languages|Interests|Projects|Certifications)\s*$"); } private static StructuredCvProfile BuildStructuredCvFromNormalizedMarkdown(string text) diff --git a/docs/phase-2-1-cv-extraction-review.md b/docs/phase-2-1-cv-extraction-review.md new file mode 100644 index 0000000..d74ccb0 --- /dev/null +++ b/docs/phase-2-1-cv-extraction-review.md @@ -0,0 +1,160 @@ +# Phase 2.1 — CV extraction: current implementation, weaknesses, proposal + +> 2026-07-20. Investigation deliverable for Phase 2.1. Grounded in the code and in a real benchmark +> CV (Connor Babbington, Systems Developer). No code changed yet — this is the "explain → identify → +> propose" step before any implementation. + +## 1. Current implementation (verified in code) + +Upload flow (`ProfileCvController`, `CvProcessingQueue`, `tools/summarizer/app.py`): + +``` +Upload file → CvUploadArtifact stored on disk, CvExtractionRun queued + ↓ (background) ProcessQueuedRunAsync +Extract text (PDF/DOCX/image OCR) + ↓ +AI /cv/normalize → LLM rewrites messy text into markdown-sectioned CV text + ↓ +split into blocks, AI /cv/classify-block per block → {section, fields, confidence} + ↓ +assemble StructuredCvProfile (+ per-field metadata: confidence, method, sourceBlockId, reviewState) + ↓ +ApplyQueuedRunResultAsync → SaveVersionAsync + user.ProfileCvStructureJson = ← REPLACE + ↓ +run.Status = "applied" (no user confirmation) +``` + +The structured model (`Models/StructuredCvProfile.cs`) already has `Contact, Summary, Jobs, +Education, Certifications, Projects, Skills, Languages` and per-field confidence metadata. Confidence +is **captured and displayed** (the review chips) but used for nothing else. + +## 2. Weaknesses (code-verified, then benchmark-verified) + +### Architectural (the big two) + +- **A1 — Replace, not merge.** `ApplyQueuedRunResultAsync` overwrites the entire profile every run + (`SaveVersionAsync(structuredCv)` + `user.ProfileCvStructureJson = structuredJson`). Re-importing a + CV, or a slightly worse OCR pass, **discards** whatever the user curated. This is the exact failure + the vision names: the profile oscillates instead of getting richer. There is no dedup, no + field-level merge, no "keep the better value". +- **A2 — No review-before-apply.** The run auto-applies (`Status = "applied"`). The user never sees + "we found 4 experiences, 15 skills, ⚠ 1 language" and never approves. The confidence metadata that + would drive such a screen is already computed and then ignored. + +### Extraction quality (verified against the benchmark CV) + +- **Q1 — Projects and Certifications are dropped.** The `/cv/normalize` heading list + (`app.py:566-573`) and the `/cv/classify-block` section enum (`app.py:622`) include only + Contact / Summary / Work Experience / Education / Skills / Languages / Interests — **no Projects, no + Certifications** — even though the data model and `StructuredCvProfileJson` fully support them. The + benchmark CV's entire **Projects** section (JobTrack, InboxIntel, infra lab) is lost. +- **Q2 — Languages in prose are missed.** The benchmark states languages only inside the summary + ("Native English speaker; Norwegian at B1"). There is no `# Languages` block for the classifier to + pick up, and normalize won't synthesise one from prose, so **English/Norwegian + levels are lost** + as structured languages. +- **Q3 — Grouped skills leak their category label.** "Development: C#, .NET, Python…" — the classifier + turns skill lines into items but nothing strips the "Development:" / "DevOps & Infrastructure:" / + "Practices:" prefixes, so a skill like "Development: C#" or a junk "Development" item can appear. +- **Q4 — Glued date/title runs.** Text extraction yields "2015–2023System Developer — Warwickshire + County Council, UK" with no space between the date range and the title. The classifier expects clean + `start`/`end`; a two-hop normalize→classify can mis-split or drop the date. +- **Q5 — Nested "Earlier roles (part-time)" list.** Three secondary jobs (Royal Vapes, The Hodcarrier, + Nuffield Health) sit as sub-bullets under a heading, not as standard entries. They are likely + mis-classified as bullets of the parent job or dropped. +- **Q6 — Two-hop LLM loses whole-CV context.** normalize (rewrites text, can hallucinate/omit) then + per-block classify (no cross-block view) means duplicate or mis-sectioned entries and no dedup. +- **Q7 — Encoding.** PDF text extraction returns mojibake for `ø`, en-dashes and apostrophes + (`T�nsberg`, `years�`, `2015�2023`) depending on the extractor. Downstream this corrupts company + names, locations and dates. (The DB round-trip for `æøå` is already fixed; this is the *extraction* + side.) + +### Confidence + +- **C1 — Confidence is display-only.** It is computed per field and shown as chips, but never used to + (a) gate what auto-applies, (b) flag low-confidence items for review, or (c) decide merge-vs-keep. + +## 3. Proposed architecture + +Keep the strengths (text extraction, per-field confidence metadata, the structured model) and add the +two missing layers plus targeted extraction fixes. + +``` +Extract → AI structured extraction (improved) → Validate → Normalize + → DIFF against current profile (new) + → REVIEW screen: "We found …" (new, confidence-aware) + → MERGE on accept (new: field-level, dedup, confidence-gated) ← never a blind replace +``` + +- **Merge engine (A1).** On accept, merge per entity: match experiences/education/projects by a stable + key (company+title+dates / institution+qualification / project name), update a field only when the + incoming confidence is high **and** differs, add genuinely new items, and never delete a curated + item the import didn't mention. Skills/languages dedup case-insensitively; language levels update + only on higher confidence. +- **Review screen (A2).** Reuse the diff: show counts and per-category adds/updates with ✓ (high) and + ⚠ (low-confidence) markers, and Accept all / Review individually / Discard. Nothing writes until + accept. (This is also Phase 2's "Import CV review" screen from the workspace refactor — one build + serves both.) +- **Extraction fixes (Q1–Q7).** Add Projects + Certifications to both prompts and the block assembler; + synthesise Languages from summary prose (or a dedicated language pass); strip skill-group prefixes; + harden date/title splitting; handle the "earlier/part-time roles" pattern; fix extraction-time + encoding. +- **Confidence gating (C1).** Drive the review markers and the merge rules from the existing + per-field confidence — no new scoring needed to start, just *use* it. + +## 4. Recommended increment order (small, verified, deployable each) + +1. **2.1-a — Merge instead of replace + review gate** (highest value, the vision's core). Backend + diff+merge engine with tests; frontend review screen. Nothing auto-overwrites again. +2. **2.1-b — Extraction coverage: Projects, Certifications, Languages-from-prose** (Q1, Q2). Prompt + + assembler + parser, benchmarked on the CV. **— DELIVERED 2026-07-20.** +3. **2.1-c — Extraction cleanup: skill-group prefixes, glued dates, part-time roles, encoding** + (Q3–Q5, Q7). +4. **2.1-d — Confidence-driven review markers and merge gating** (C1), once the review screen exists. + +Each ships independently and leaves production green. + +## 5. Benchmark as regression fixture + +Save the benchmark CV's expected structured output as a test fixture: 5 experiences (2 primary + 3 +part-time) or a documented decision on the part-time roles, 3 projects, grouped skills flattened, +English (Native) + Norwegian (B1) languages, education entry, contact with Norwegian location intact. +Extraction changes are measured against it — without overfitting (the rules must generalise). + +## 2.1-b delivered — extraction coverage (2026-07-20) + +**What changed.** The gap was upstream only: the C# assembler (`StructuredCvProfileJson.FromSections` ++ `BuildStructuredCvFromNormalizedMarkdown`) already maps `Projects`, `Certifications` and `Languages` +headings — the AI `normalize` prompt just never emitted them, so they were dropped. + +- `tools/summarizer/app.py` `/cv/normalize`: added `# Projects` and `# Certifications` headings with + shapes; added a **languages-from-prose** rule (extract "native English", "Norwegian B1" from the + summary even without a Languages section; ignore programming languages); added **skill-group prefix + stripping** ("Development:", "DevOps & Infrastructure:", "Practices:" are dropped, only the skills + remain). +- `/cv/classify-block`: added `Projects` and `Certifications` to the section enum + rules (fallback + path). +- `ProfileCvController.LooksLikeNormalizedMarkdownCv`: recognises `# Projects` / `# Certifications` so + a CV whose structured content is mostly those sections still takes the markdown path. + +**Verification.** 4 new backend tests (`CvExtractionCoverageTests`) lock the C# mapping of +Projects/Certifications/Languages; 1 new ai-service test (`test_classify_block_supports_projects_section`). +426 backend tests and 17 ai-service tests pass; `app.py` compiles. The LLM behaviour itself +(prompt → headings) could not be run here (no Ollama), but the C# side that consumes the headings is +proven, and the prompt change is additive/contract-safe. + +**Deployment note.** These prompt changes live in the **ai-service container**, which `deploy.sh` +does **not** rebuild by default. Deploy with `DEPLOY_BUILD_AI_SERVICE=true ./deploy/deploy.sh` (or +rebuild `ai-service` manually) or the extraction change won't take effect. No database or backend +schema change. + +**Not done here (moved to 2.1-c):** deterministic C# safety-nets for skill-prefix stripping and +glued-date splitting, and the "earlier/part-time roles" pattern. 2.1-b relies on the prompt for those; +2.1-c hardens them deterministically. + +## 6. Open product decisions (need a call before building) + +- **Merge matching keys** — how aggressively to treat two experiences as "the same" (company+title vs + fuzzy). Conservative (fewer merges, some dupes) vs aggressive (cleaner, risk of wrong merges). +- **Part-time/earlier roles** — separate experience entries, or a sub-list on the primary role? +- **Auto-apply threshold** — does anything ever apply without review (e.g. an empty profile's first + import), or is review always required? diff --git a/tools/summarizer/app.py b/tools/summarizer/app.py index 30fd10a..12af498 100644 --- a/tools/summarizer/app.py +++ b/tools/summarizer/app.py @@ -569,6 +569,8 @@ Rules for normalized_text: # Work Experience # Education # Skills + # Projects + # Certifications # Languages # Interests - Under # Contact, put one plain value per line, no labels unless unavoidable: @@ -591,7 +593,19 @@ Rules for normalized_text: Institution, Location line 2016 - 2019 line - detail -- Under # Skills and # Languages, use one bullet per item. +- Under # Projects, for each project use this exact shape (blank line between projects): + Project name + - one short description line covering what it is and the tech used +- Under # Certifications, one certification per line: name, then issuer and year if stated. +- Under # Skills, use one bullet per item. If skills are grouped with a category label such as + "Development:", "DevOps & Infrastructure:" or "Practices:", DROP the category label and list only + the individual skills as separate bullets. Never keep the category word as a skill. +- Under # Languages, one language per line as "Name: Level" (e.g. "English: Native", "Norwegian: B1"). + IMPORTANT: languages are often stated only inside the summary or profile text (e.g. "native English + speaker", "Norwegian at B1"). When you see a spoken/written human language and any proficiency + (native, fluent, C1, B2, B1, A2, conversational, basic), add it here even if there is no dedicated + languages section in the source. Do NOT treat programming languages (C#, Python, JavaScript, SQL) as + human languages. - Remove OCR/layout noise. - Do not output placeholders like Not specified. - If uncertain, omit the field/line rather than invent. @@ -619,7 +633,7 @@ async def classify_cv_block(req: CvClassifyBlockRequest): You classify one CV text block into structured JSON. Return ONLY valid JSON with this exact shape: {{ - "section": "Contact|Professional Summary|Work Experience|Education|Skills|Languages|Interests|Other", + "section": "Contact|Professional Summary|Work Experience|Education|Skills|Projects|Certifications|Languages|Interests|Other", "confidence": 0.0, "reason": "short reason", "title": string|null, @@ -636,7 +650,9 @@ Rules: - Preserve facts only. - section must be one of the listed values. - Use Work Experience only for job/employment blocks. -- Use Education only for degree/course/certification blocks. +- Use Education only for degree/diploma/course blocks. +- Use Projects for personal/side/portfolio project blocks (put the project name in title and details in bullets). +- Use Certifications for named certifications/licences (put the certification name in title). - For Contact blocks, keep title/company/start/end null and bullets/summary/skills empty. - For Professional Summary blocks, prefer summary for concise summary lines and keep bullets empty unless the source is already bullet-like. - For Skills blocks, prefer skills for normalized skill items and keep title/company/start/end null. diff --git a/tools/summarizer/tests/test_app.py b/tools/summarizer/tests/test_app.py index e8e6a5f..fd1a56b 100644 --- a/tools/summarizer/tests/test_app.py +++ b/tools/summarizer/tests/test_app.py @@ -134,6 +134,38 @@ def test_classify_block_returns_structured_json(monkeypatch): assert payload["skills"] == ["Python", "SQL"] +def test_classify_block_supports_projects_section(monkeypatch): + # Phase 2.1-b: Projects and Certifications are now valid classified sections so project blocks + # (e.g. the benchmark CV's JobTrack/InboxIntel) are no longer dropped into "Other". + module = load_app_module(monkeypatch) + + def fake_generate_json(prompt: str): + assert "Projects" in prompt # the enum now advertises Projects to the model + return { + "section": "Projects", + "confidence": 0.83, + "reason": "project block", + "title": "JobTrack", + "company": None, + "location": None, + "start": None, + "end": None, + "bullets": ["Full-stack job-application tracker (React, ASP.NET Core, SQLite, Docker)."], + "summary": [], + "skills": [], + } + + monkeypatch.setattr(module, "_ollama_generate_json", fake_generate_json) + client = TestClient(module.app) + + response = client.post("/cv/classify-block", json={"block": "JobTrack - Full-stack job-application tracker (React, ASP.NET Core, SQLite, Docker)."}) + + assert response.status_code == 200 + payload = response.json() + assert payload["section"] == "Projects" + assert payload["title"] == "JobTrack" + + def test_classify_block_defaults_missing_section_to_other(monkeypatch): module = load_app_module(monkeypatch) monkeypatch.setattr(module, "_ollama_generate_json", lambda prompt: {"bullets": []})