From 0d658358575abc4f30aed97938fd3e20e40db443 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Wed, 1 Apr 2026 12:25:45 +0200 Subject: [PATCH] feat: add cv benchmark workflow and admin visibility --- JobTrackerApi.Tests/CvCorpusHarnessTests.cs | 233 +++++++++++++++-- .../JobApplicationsControllerTests.cs | 2 +- .../Controllers/AdminSystemController.cs | 21 ++ JobTrackerApi/Services/AppPaths.cs | 8 + JobTrackerApi/Services/SummarizerService.cs | 23 ++ docs/cv-builder-parser-benchmark.md | 224 +++++++++++++++++ job-tracker-ui/src/admin-system-page.test.tsx | 20 ++ job-tracker-ui/src/pages/AdminSystemPage.tsx | 42 +++- job-tracker-ui/src/pages/ProfilePage.tsx | 238 +++++++++++++----- job-tracker-ui/src/profile-page.test.tsx | 12 +- job-tracker-ui/src/profileCv.ts | 44 ++++ job-tracker-ui/src/types.ts | 1 + scripts/run-cv-benchmark.sh | 23 ++ tools/summarizer/README.md | 2 +- tools/summarizer/app.py | 32 ++- tools/summarizer/tests/test_app.py | 2 + 16 files changed, 832 insertions(+), 95 deletions(-) create mode 100644 docs/cv-builder-parser-benchmark.md create mode 100755 scripts/run-cv-benchmark.sh diff --git a/JobTrackerApi.Tests/CvCorpusHarnessTests.cs b/JobTrackerApi.Tests/CvCorpusHarnessTests.cs index c36aa55..e9ced3c 100644 --- a/JobTrackerApi.Tests/CvCorpusHarnessTests.cs +++ b/JobTrackerApi.Tests/CvCorpusHarnessTests.cs @@ -1,4 +1,5 @@ using System.Reflection; +using System.Security.Cryptography; using System.Text.Json; using JobTrackerApi.Controllers; using JobTrackerApi.Models; @@ -29,11 +30,19 @@ public sealed class CvCorpusHarnessTests || path.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) || path.EndsWith(".md", StringComparison.OrdinalIgnoreCase)) .OrderBy(path => path, StringComparer.OrdinalIgnoreCase) - .Take(8) .ToList(); if (files.Count == 0) return; + var outputRoot = ResolveOutputRoot(); + var outputsDir = Path.Combine(outputRoot, "outputs"); + var candidateFixturesDir = Path.Combine(outputRoot, "candidate-fixtures"); + var approvedFixturesDir = ResolveApprovedFixturesRoot(outputRoot); + Directory.CreateDirectory(outputRoot); + Directory.CreateDirectory(outputsDir); + Directory.CreateDirectory(candidateFixturesDir); + Directory.CreateDirectory(approvedFixturesDir); + var user = new ApplicationUser { Id = "user-1", ProfileCvText = "seed" }; var userManager = TestHostFactory.CreateUserManager(); userManager.Setup(x => x.GetUserAsync(It.IsAny())).ReturnsAsync(user); @@ -44,8 +53,8 @@ public sealed class CvCorpusHarnessTests aiService.Setup(x => x.SummarizeSectionAsync(It.Is(instruction => instruction.Contains("Reconstruct this CV text extracted from a PDF", StringComparison.Ordinal)), It.IsAny(), 2800, 900)).ReturnsAsync((string _, string text, int _, int __) => text); await using var db = TestHostFactory.CreateInMemoryDb(); - var paths = CreatePaths(); - var controller = new ProfileCvController(userManager.Object, aiService.Object, db, paths, NoOpCvAiClassifier.Instance) + var paths = CreatePaths(outputRoot); + var controller = new ProfileCvController(userManager.Object, aiService.Object, db, paths, null, NoOpCvAiClassifier.Instance) { ControllerContext = new ControllerContext { HttpContext = new DefaultHttpContext() } }; @@ -55,7 +64,7 @@ public sealed class CvCorpusHarnessTests Assert.NotNull(extractMethod); Assert.NotNull(buildMethod); - var report = new List(); + var entries = new List(); foreach (var path in files) { await using var stream = File.OpenRead(path); @@ -72,31 +81,212 @@ public sealed class CvCorpusHarnessTests Assert.False(string.IsNullOrWhiteSpace(text)); var buildTask = (Task)buildMethod!.Invoke(controller, new object[] { text, CancellationToken.None })!; - var structured = await buildTask; + var structured = StructuredCvProfileJson.Normalize(await buildTask); Assert.NotNull(structured); - report.Add(new + var slug = Slugify(fileName); + var normalizedJson = StructuredCvProfileJson.Serialize(structured); + var outputPath = Path.Combine(outputsDir, $"{slug}.json"); + await File.WriteAllTextAsync(outputPath, PrettyJson(normalizedJson)); + + var approvedPath = Path.Combine(approvedFixturesDir, $"{slug}.json"); + var candidateFixturePath = Path.Combine(candidateFixturesDir, $"{slug}.json"); + string? diffSummary = null; + var approvedExists = File.Exists(approvedPath); + if (approvedExists) { - file = fileName, - characters = text.Length, - contactLocation = structured.Contact.Location, - firstJob = structured.Jobs.FirstOrDefault()?.Title, - firstJobLocation = structured.Jobs.FirstOrDefault()?.Location, - firstEducation = structured.Education.FirstOrDefault()?.Qualification, - firstEducationLocation = structured.Education.FirstOrDefault()?.Location, - suspiciousLocations = structured.Jobs.Select(job => job.Location) + var approvedJson = await File.ReadAllTextAsync(approvedPath); + diffSummary = SummarizeDiff(approvedJson, normalizedJson); + } + else + { + await File.WriteAllTextAsync(candidateFixturePath, PrettyJson(normalizedJson)); + diffSummary = "No approved fixture yet — candidate fixture written."; + } + + entries.Add(new CvBenchmarkEntry( + FileName: fileName, + Slug: slug, + Extension: extension, + Characters: text.Length, + OutputPath: outputPath, + ApprovedFixturePath: approvedExists ? approvedPath : null, + CandidateFixturePath: approvedExists ? null : candidateFixturePath, + ContactLocation: structured.Contact.Location, + FirstJob: structured.Jobs.FirstOrDefault()?.Title, + FirstJobLocation: structured.Jobs.FirstOrDefault()?.Location, + FirstEducation: structured.Education.FirstOrDefault()?.Qualification, + FirstEducationLocation: structured.Education.FirstOrDefault()?.Location, + QualificationLevels: structured.Education.Select(x => x.QualificationLevel).Where(x => !string.IsNullOrWhiteSpace(x)).Cast().ToList(), + SuspiciousLocations: structured.Jobs.Select(job => job.Location) .Concat(structured.Education.Select(education => education.Location)) .Append(structured.Contact.Location) .Where(value => !string.IsNullOrWhiteSpace(value)) + .Cast() .Where(LooksSuspiciousLocation) - .ToList() - }); + .ToList(), + CoverageScore: ComputeCoverageScore(structured), + ConfidenceScore: ComputeConfidenceScore(structured), + ConsistencyScore: ComputeConsistencyScore(structured), + DiffSummary: diffSummary + )); } - var reportPath = Path.Combine(Path.GetTempPath(), $"jobtracker-cv-corpus-{DateTime.UtcNow:yyyyMMddHHmmss}.json"); - await File.WriteAllTextAsync(reportPath, JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true })); + var summary = new CvBenchmarkSummary( + CorpusRoot, + outputRoot, + DateTimeOffset.UtcNow, + entries.Count, + Math.Round(entries.Average(x => x.CoverageScore), 3), + Math.Round(entries.Average(x => x.ConfidenceScore), 3), + Math.Round(entries.Average(x => x.ConsistencyScore), 3), + entries.Count(x => x.SuspiciousLocations.Count > 0), + entries.Count(x => x.ApprovedFixturePath is null), + entries + ); - Assert.True(report.Count > 0); + var indexPath = Path.Combine(outputRoot, "index.json"); + var reportPath = Path.Combine(outputRoot, "report.md"); + await File.WriteAllTextAsync(indexPath, JsonSerializer.Serialize(summary, new JsonSerializerOptions { WriteIndented = true })); + await File.WriteAllTextAsync(reportPath, RenderMarkdownReport(summary)); + + Assert.True(entries.Count > 0); + } + + private sealed record CvBenchmarkEntry( + string FileName, + string Slug, + string Extension, + int Characters, + string OutputPath, + string? ApprovedFixturePath, + string? CandidateFixturePath, + string? ContactLocation, + string? FirstJob, + string? FirstJobLocation, + string? FirstEducation, + string? FirstEducationLocation, + List QualificationLevels, + List SuspiciousLocations, + double CoverageScore, + double ConfidenceScore, + double ConsistencyScore, + string? DiffSummary); + + private sealed record CvBenchmarkSummary( + string CorpusRoot, + string OutputRoot, + DateTimeOffset GeneratedAtUtc, + int TotalFiles, + double AverageCoverage, + double AverageConfidence, + double AverageConsistency, + int FilesWithSuspiciousLocations, + int MissingApprovedFixtures, + List Entries); + + private static string ResolveOutputRoot() + { + var configured = Environment.GetEnvironmentVariable("CV_BENCHMARK_OUTPUT_DIR"); + if (!string.IsNullOrWhiteSpace(configured)) return configured.Trim(); + return Path.Combine(Path.GetTempPath(), "jobtracker-cv-benchmark", DateTime.UtcNow.ToString("yyyyMMddHHmmss")); + } + + private static string ResolveApprovedFixturesRoot(string outputRoot) + { + var configured = Environment.GetEnvironmentVariable("CV_BENCHMARK_APPROVED_DIR"); + if (!string.IsNullOrWhiteSpace(configured)) return configured.Trim(); + return Path.Combine(outputRoot, "approved-fixtures"); + } + + private static string PrettyJson(string normalizedJson) + { + using var doc = JsonDocument.Parse(normalizedJson); + return JsonSerializer.Serialize(doc.RootElement, new JsonSerializerOptions { WriteIndented = true }); + } + + private static string SummarizeDiff(string approvedJson, string actualJson) + { + if (JsonDocument.Parse(approvedJson).RootElement.ToString() == JsonDocument.Parse(actualJson).RootElement.ToString()) + { + return "Matches approved fixture."; + } + + var approvedHash = Hash(approvedJson); + var actualHash = Hash(actualJson); + return $"Fixture differs (approved {approvedHash[..8]}, actual {actualHash[..8]})."; + } + + private static string Hash(string value) => Convert.ToHexString(SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(value))).ToLowerInvariant(); + + private static double ComputeCoverageScore(StructuredCvProfile structured) + { + var signals = new[] + { + !string.IsNullOrWhiteSpace(structured.Contact.FullName), + !string.IsNullOrWhiteSpace(structured.Contact.Email), + !string.IsNullOrWhiteSpace(structured.Contact.Location), + structured.Summary.Count > 0, + structured.Skills.Count > 0, + structured.Jobs.Count > 0, + structured.Education.Count > 0, + structured.Certifications.Count > 0 || structured.Projects.Count > 0 || structured.OtherSections.Count > 0, + }; + return signals.Count(x => x) / (double)signals.Length; + } + + private static double ComputeConfidenceScore(StructuredCvProfile structured) + { + var confidences = structured.Metadata.Fields.Values.Select(x => x.Confidence).Where(x => x.HasValue).Select(x => x!.Value).ToList(); + return confidences.Count == 0 ? 0.55 : Math.Clamp(confidences.Average(), 0, 1); + } + + private static double ComputeConsistencyScore(StructuredCvProfile structured) + { + var penalties = 0; + penalties += structured.Jobs.Count(job => LooksSuspiciousLocation(job.Location)); + penalties += structured.Education.Count(education => LooksSuspiciousLocation(education.Location)); + penalties += LooksSuspiciousLocation(structured.Contact.Location) ? 1 : 0; + penalties += structured.Education.Count(education => string.IsNullOrWhiteSpace(education.QualificationLevel) && !string.IsNullOrWhiteSpace(education.Qualification)); + return Math.Max(0, 1 - (penalties * 0.12)); + } + + private static string RenderMarkdownReport(CvBenchmarkSummary summary) + { + var lines = new List + { + "# CV benchmark report", + string.Empty, + $"- Generated: {summary.GeneratedAtUtc:O}", + $"- Corpus root: `{summary.CorpusRoot}`", + $"- Output root: `{summary.OutputRoot}`", + $"- Files: {summary.TotalFiles}", + $"- Average coverage: {summary.AverageCoverage:P0}", + $"- Average confidence: {summary.AverageConfidence:P0}", + $"- Average consistency: {summary.AverageConsistency:P0}", + $"- Files with suspicious locations: {summary.FilesWithSuspiciousLocations}", + $"- Missing approved fixtures: {summary.MissingApprovedFixtures}", + string.Empty, + "| File | Coverage | Confidence | Consistency | Suspicious locations | Fixture |", + "|---|---:|---:|---:|---:|---|", + }; + + lines.AddRange(summary.Entries.Select(entry => + $"| {entry.FileName} | {entry.CoverageScore:P0} | {entry.ConfidenceScore:P0} | {entry.ConsistencyScore:P0} | {entry.SuspiciousLocations.Count} | {entry.DiffSummary} |")); + + lines.Add(string.Empty); + lines.Add("## Notes"); + lines.Add("- `outputs/*.json` contains the latest normalized parser output for each CV."); + lines.Add("- `candidate-fixtures/*.json` is created when no approved fixture exists yet."); + lines.Add("- To build a regression baseline, review a candidate fixture and copy it into the approved-fixtures directory used by the runner."); + return string.Join(Environment.NewLine, lines); + } + + private static string Slugify(string value) + { + var cleaned = new string((value ?? string.Empty).ToLowerInvariant().Select(ch => char.IsLetterOrDigit(ch) ? ch : '-').ToArray()); + while (cleaned.Contains("--", StringComparison.Ordinal)) cleaned = cleaned.Replace("--", "-", StringComparison.Ordinal); + return cleaned.Trim('-'); } private static bool LooksSuspiciousLocation(string? value) @@ -119,7 +309,7 @@ public sealed class CvCorpusHarnessTests }; } - private static AppPaths CreatePaths() + private static AppPaths CreatePaths(string outputRoot) { var tempRoot = Path.Combine(Path.GetTempPath(), $"jobtracker-cv-corpus-{Guid.NewGuid():N}"); Directory.CreateDirectory(tempRoot); @@ -128,7 +318,8 @@ public sealed class CvCorpusHarnessTests .AddInMemoryCollection(new Dictionary { ["Data:Root"] = tempRoot, - ["Data:CvArtifactsRoot"] = Path.Combine(tempRoot, "CvArtifacts") + ["Data:CvArtifactsRoot"] = Path.Combine(tempRoot, "CvArtifacts"), + ["Data:CvBenchmarksRoot"] = outputRoot, }) .Build(); diff --git a/JobTrackerApi.Tests/JobApplicationsControllerTests.cs b/JobTrackerApi.Tests/JobApplicationsControllerTests.cs index 3d696ae..cde2de6 100644 --- a/JobTrackerApi.Tests/JobApplicationsControllerTests.cs +++ b/JobTrackerApi.Tests/JobApplicationsControllerTests.cs @@ -27,7 +27,7 @@ public sealed class JobApplicationsControllerTests Assert.NotNull(type); var ctor = type!.GetConstructors().Single(); - var parameters = ctor.GetParameters().Select(x => x.Name).ToArray(); + var parameters = ctor.GetParameters().Select(x => x.Name).Where(x => x is not null).Select(x => x!).ToHashSet(StringComparer.OrdinalIgnoreCase); Assert.Contains("coverLetterText", parameters); Assert.Contains("notes", parameters); } diff --git a/JobTrackerApi/Controllers/AdminSystemController.cs b/JobTrackerApi/Controllers/AdminSystemController.cs index 428ae50..6b0eb60 100644 --- a/JobTrackerApi/Controllers/AdminSystemController.cs +++ b/JobTrackerApi/Controllers/AdminSystemController.cs @@ -35,6 +35,7 @@ public sealed class AdminSystemController : ControllerBase public sealed record DatabaseStatusDto(string Provider, bool LooksConfigured, bool CanConnect, string? Target, bool UsesFileStorage, string? Warning); public sealed record RuntimeStatusDto(string Framework, string OSDescription, string ProcessArchitecture, string? MachineName); public sealed record AuthStatusDto(bool Required, bool HasJwtKey, bool GoogleConfigured, bool GmailConfigured); + public sealed record CvBenchmarkStatusDto(string? IndexJson, string? ReportMarkdown, string RootPath, DateTimeOffset? LastUpdatedAtUtc); public sealed record SystemStatusDto( string Environment, string ContentRoot, @@ -86,6 +87,22 @@ public sealed class AdminSystemController : ControllerBase return Ok(await _emailSettings.UpdateAsync(request, cancellationToken)); } + [HttpGet("cv-benchmark")] + public async Task> GetCvBenchmarkStatus(CancellationToken cancellationToken) + { + var indexPath = Path.Combine(_paths.CvBenchmarksRoot, "index.json"); + var reportPath = Path.Combine(_paths.CvBenchmarksRoot, "report.md"); + var indexJson = System.IO.File.Exists(indexPath) ? await System.IO.File.ReadAllTextAsync(indexPath, cancellationToken) : null; + var reportMarkdown = System.IO.File.Exists(reportPath) ? await System.IO.File.ReadAllTextAsync(reportPath, cancellationToken) : null; + var lastUpdated = new[] + { + System.IO.File.Exists(indexPath) ? System.IO.File.GetLastWriteTimeUtc(indexPath) : (DateTime?)null, + System.IO.File.Exists(reportPath) ? System.IO.File.GetLastWriteTimeUtc(reportPath) : (DateTime?)null, + }.Where(value => value.HasValue).Select(value => value!.Value).DefaultIfEmpty().Max(); + + return Ok(new CvBenchmarkStatusDto(indexJson, reportMarkdown, _paths.CvBenchmarksRoot, lastUpdated == default ? null : new DateTimeOffset(DateTime.SpecifyKind(lastUpdated, DateTimeKind.Utc)))); + } + [HttpGet] public async Task> Get(CancellationToken cancellationToken) { @@ -128,6 +145,10 @@ public sealed class AdminSystemController : ControllerBase OllamaReachable: null, OllamaModel: null, OllamaModelAvailable: null, + OllamaVersion: null, + OllamaInstalledModels: Array.Empty(), + OllamaLoadedModels: Array.Empty(), + OllamaLoadedCount: 0, HealthLatencyMs: null, ProbeLatencyMs: null, LastProbeAt: null, diff --git a/JobTrackerApi/Services/AppPaths.cs b/JobTrackerApi/Services/AppPaths.cs index ac83c3f..337fe43 100644 --- a/JobTrackerApi/Services/AppPaths.cs +++ b/JobTrackerApi/Services/AppPaths.cs @@ -9,6 +9,7 @@ namespace JobTrackerApi.Services public string AttachmentsRoot { get; } public string CvArtifactsRoot { get; } public string CvExportsRoot { get; } + public string CvBenchmarksRoot { get; } public AppPaths(IConfiguration cfg, IHostEnvironment env) { @@ -39,6 +40,13 @@ namespace JobTrackerApi.Services Directory.CreateDirectory(cvExportsRoot); CvExportsRoot = cvExportsRoot; + + var cvBenchmarksRoot = (cfg["Data:CvBenchmarksRoot"] ?? "").Trim(); + if (string.IsNullOrWhiteSpace(cvBenchmarksRoot)) cvBenchmarksRoot = Path.Combine(DataRoot, "CvBenchmarks"); + if (!Path.IsPathRooted(cvBenchmarksRoot)) cvBenchmarksRoot = Path.Combine(env.ContentRootPath, cvBenchmarksRoot); + + Directory.CreateDirectory(cvBenchmarksRoot); + CvBenchmarksRoot = cvBenchmarksRoot; } public string GetDbPath(string fileName = "jobtracker.db") => Path.Combine(DataRoot, fileName); diff --git a/JobTrackerApi/Services/SummarizerService.cs b/JobTrackerApi/Services/SummarizerService.cs index 6ebd044..1ae180c 100644 --- a/JobTrackerApi/Services/SummarizerService.cs +++ b/JobTrackerApi/Services/SummarizerService.cs @@ -25,6 +25,10 @@ namespace JobTrackerApi.Services bool? OllamaReachable, string? OllamaModel, bool? OllamaModelAvailable, + string? OllamaVersion, + IReadOnlyList? OllamaInstalledModels, + IReadOnlyList? OllamaLoadedModels, + int? OllamaLoadedCount, double? HealthLatencyMs, double? ProbeLatencyMs, DateTimeOffset? LastProbeAt, @@ -66,6 +70,7 @@ namespace JobTrackerApi.Services public interface ISummarizerService : IAiService { + new Task SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40); } public class SummarizerService : ISummarizerService @@ -318,6 +323,10 @@ namespace JobTrackerApi.Services bool? ollamaReachable = null; string? ollamaModel = null; bool? ollamaModelAvailable = null; + string? ollamaVersion = null; + List? ollamaInstalledModels = null; + List? ollamaLoadedModels = null; + int? ollamaLoadedCount = null; double? healthLatencyMs = null; var healthy = false; string? healthError = null; @@ -344,6 +353,16 @@ namespace JobTrackerApi.Services if (doc.RootElement.TryGetProperty("ollama_reachable", out var ollamaReachableEl) && ollamaReachableEl.ValueKind is JsonValueKind.True or JsonValueKind.False) ollamaReachable = ollamaReachableEl.GetBoolean(); if (doc.RootElement.TryGetProperty("ollama_model", out var ollamaModelEl)) ollamaModel = ollamaModelEl.GetString(); if (doc.RootElement.TryGetProperty("ollama_model_available", out var ollamaModelAvailableEl) && ollamaModelAvailableEl.ValueKind is JsonValueKind.True or JsonValueKind.False) ollamaModelAvailable = ollamaModelAvailableEl.GetBoolean(); + if (doc.RootElement.TryGetProperty("ollama_version", out var ollamaVersionEl)) ollamaVersion = ollamaVersionEl.GetString(); + if (doc.RootElement.TryGetProperty("ollama_installed_models", out var ollamaInstalledModelsEl) && ollamaInstalledModelsEl.ValueKind == JsonValueKind.Array) + { + ollamaInstalledModels = ollamaInstalledModelsEl.EnumerateArray().Where(x => x.ValueKind == JsonValueKind.String).Select(x => x.GetString()).Where(x => !string.IsNullOrWhiteSpace(x)).Cast().ToList(); + } + if (doc.RootElement.TryGetProperty("ollama_loaded_models", out var ollamaLoadedModelsEl) && ollamaLoadedModelsEl.ValueKind == JsonValueKind.Array) + { + ollamaLoadedModels = ollamaLoadedModelsEl.EnumerateArray().Where(x => x.ValueKind == JsonValueKind.String).Select(x => x.GetString()).Where(x => !string.IsNullOrWhiteSpace(x)).Cast().ToList(); + } + if (doc.RootElement.TryGetProperty("ollama_loaded_count", out var ollamaLoadedCountEl) && ollamaLoadedCountEl.ValueKind == JsonValueKind.Number) ollamaLoadedCount = ollamaLoadedCountEl.GetInt32(); } else { @@ -406,6 +425,10 @@ namespace JobTrackerApi.Services OllamaReachable: ollamaReachable, OllamaModel: ollamaModel, OllamaModelAvailable: ollamaModelAvailable, + OllamaVersion: ollamaVersion, + OllamaInstalledModels: ollamaInstalledModels, + OllamaLoadedModels: ollamaLoadedModels, + OllamaLoadedCount: ollamaLoadedCount, HealthLatencyMs: healthLatencyMs, ProbeLatencyMs: probeLatencyMs, LastProbeAt: lastProbeAt, diff --git a/docs/cv-builder-parser-benchmark.md b/docs/cv-builder-parser-benchmark.md new file mode 100644 index 0000000..2393264 --- /dev/null +++ b/docs/cv-builder-parser-benchmark.md @@ -0,0 +1,224 @@ +# CV builder, parser, benchmark, and Ollama admin integration + +## What changed + +This branch upgrades the Profile CV flow from a text-only rewrite helper into a template-driven CV builder backed by the server-side renderer/PDF pipeline, strengthens CV normalization around location and qualification handling, adds a repeatable local corpus benchmark workflow, and expands the admin system page with richer Ollama visibility. + +## Profile CV builder + +### New backend capabilities + +`JobTrackerApi/Controllers/ProfileCvController.cs` + +- Hardened `POST /api/profile-cv/rewrite-section` + - accepts flexible `jobApplicationId` payloads (number or blank string) + - uses richer saved-job context for tailoring + - logs empty AI responses with useful context +- Added `GET /api/profile-cv/templates` +- Added `POST /api/profile-cv/rewrite-preview` + - rewrites either the whole CV or one selected section + - rebuilds structured CV from the rewritten full text + - maps the result into the shared template renderer + - returns rendered HTML, file name, rewritten text, and full replacement text +- Added `POST /api/profile-cv/export-pdf` + - uses the same rendered HTML and the shared Playwright exporter + +### Frontend flow + +`job-tracker-ui/src/pages/ProfilePage.tsx` + +- Replaced the old rewrite draft box with a template-driven builder section. +- Users can: + - choose from 6 templates + - optionally target one section + - target by free-text role or saved job + - inspect the rewritten content + - inspect the actual rendered preview + - download a PDF + - replace the master CV with the rebuilt full-text result + +## Templates + +Shared renderer: `JobTrackerApi/Services/CvTemplateRenderer.cs` + +Available templates: +- `ats-minimal` +- `harvard` +- `auckland` +- `edinburgh` +- `monarch` +- `fjord` + +### Adding a new template + +1. Add the new template id to `NormalizeTemplateId()` in: + - `JobTrackerApi/Services/CvTemplateRenderer.cs` + - `JobTrackerApi/Controllers/ProfileCvController.cs` +2. Add a render branch in `CvTemplateRenderer.Render()`. +3. Add a descriptor to `GetCvTemplateDescriptors()`. +4. Add the matching card entry in `job-tracker-ui/src/pages/ProfilePage.tsx` if you want a custom preview card. + +## PDF generation + +The master CV builder now reuses the existing server-side pipeline: + +1. rewrite full text / section +2. rebuild structured CV +3. map to `TailoredCvDocument` +4. render HTML via `ICvTemplateRenderer` +5. export PDF via `ICvPdfExporter` / Playwright + +This keeps PDF output visually aligned with the selected template and avoids a separate client-only print implementation. + +## Parser and structured CV changes + +### Shared schema + +`Models/StructuredCvProfile.cs` + +Added: +- `education[].qualificationLevel` +- top-level `certifications[]` +- top-level `projects[]` + +`qualification` remains the original preserved text. + +### Normalization improvements + +`Models/StructuredCvProfileJson.cs` + +- tighter location sanitization to avoid skill or role spillover into location fields +- qualification level normalization to one of: + - `Secondary` + - `Diploma/Certificate` + - `Bachelor` + - `Master` + - `PhD` + - `Other` +- first-class normalization for certifications and projects +- section reconstruction now includes certifications and projects + +### Extraction prompt improvements + +`JobTrackerApi/Controllers/ProfileCvController.cs` + +The LLM extraction prompt now explicitly asks for: +- qualification level enum +- certifications +- projects +- strict location separation rules +- preservation of original qualification text + +## Benchmark workflow + +### Runner + +Use: + +```bash +./scripts/run-cv-benchmark.sh +``` + +Optional overrides: + +```bash +CV_BENCHMARK_OUTPUT_DIR=/absolute/output/path \ +CV_BENCHMARK_APPROVED_DIR=/absolute/approved/fixtures/path \ +./scripts/run-cv-benchmark.sh +``` + +### Inputs + +The runner scans: + +- `/home/pi/cvs` + +Supported corpus file types: +- PDF +- DOCX +- TXT +- MD + +### Outputs + +The runner writes: + +- `index.json` — machine-readable summary +- `report.md` — markdown overview +- `outputs/*.json` — latest normalized structured output per CV +- `candidate-fixtures/*.json` — created when no approved fixture exists yet + +Approved fixtures are local by design and should be reviewed manually before being promoted into the approved fixture path you use for regression comparisons. + +### Admin review + +`GET /api/admin/system/cv-benchmark` + +The admin system page surfaces: +- benchmark root path +- last benchmark update time +- latest markdown summary + +## Ollama admin visibility + +### Python health endpoint + +`tools/summarizer/app.py` + +`GET /health` now returns additional Ollama metadata when configured/reachable: +- `ollama_version` +- `ollama_installed_models` +- `ollama_loaded_models` +- `ollama_loaded_count` + +### Backend propagation + +`JobTrackerApi/Services/SummarizerService.cs` + +The backend metrics shape now carries those fields through to admin consumers. + +### Admin UI + +`job-tracker-ui/src/pages/AdminSystemPage.tsx` + +The system page now shows: +- Ollama version +- loaded model count +- installed model chips +- loaded model chips +- benchmark summary panel + +## Verification used on this branch + +### Backend + +```bash +dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --filter ProfileCvControllerTests +dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --filter "ProfileCvControllerTests|AuthAndSystemControllerTests|JobApplicationsApplicationPackageTests" +dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --filter CvCorpusHarnessTests +``` + +### Frontend + +```bash +cd job-tracker-ui && CI=true npm test -- --runInBand --watch=false src/profile-page.test.tsx +cd job-tracker-ui && CI=true npm test -- --runInBand --watch=false src/admin-system-page.test.tsx +cd job-tracker-ui && CI=true npm test -- --runInBand --watch=false src/profile-page.test.tsx src/admin-system-page.test.tsx src/job-details-generated-drafts.test.tsx +``` + +### Benchmark runner + +```bash +CV_BENCHMARK_OUTPUT_DIR="$(pwd)/tmp/cv-benchmarks/latest" \ +CV_BENCHMARK_APPROVED_DIR="$(pwd)/tmp/cv-benchmarks/approved" \ +./scripts/run-cv-benchmark.sh +``` + +### Python service tests + +The summarizer Python unit tests were updated for the new health payload, but this machine currently lacks `pip` / `venv` support (`python3 -m venv` fails because `python3.12-venv` is not installed), so test execution is environment-blocked here. Once Python packaging is available, run: + +```bash +cd tools/summarizer +python3 -m pytest -q tests/test_app.py +``` diff --git a/job-tracker-ui/src/admin-system-page.test.tsx b/job-tracker-ui/src/admin-system-page.test.tsx index c3560ec..9c4c3b1 100644 --- a/job-tracker-ui/src/admin-system-page.test.tsx +++ b/job-tracker-ui/src/admin-system-page.test.tsx @@ -43,6 +43,14 @@ describe('AdminSystemPage', () => { gpuName: null, ocrAvailable: true, ocrLanguages: 'eng', + ollamaConfigured: true, + ollamaReachable: true, + ollamaModel: 'qwen2.5:7b', + ollamaModelAvailable: true, + ollamaVersion: '0.7.0', + ollamaInstalledModels: ['qwen2.5:7b', 'nomic-embed-text'], + ollamaLoadedModels: ['qwen2.5:7b'], + ollamaLoadedCount: 1, healthLatencyMs: 12.4, probeLatencyMs: 25.8, lastProbeAt: '2026-03-23T10:00:00Z', @@ -82,6 +90,15 @@ describe('AdminSystemPage', () => { }, } as any); } + if (url === '/admin/system/cv-benchmark') { + return Promise.resolve({ + data: { + rootPath: '/data/CvBenchmarks/latest', + lastUpdatedAtUtc: '2026-03-23T10:10:00Z', + reportMarkdown: '# CV benchmark report\n\n- Files: 4', + }, + } as any); + } return Promise.resolve({ data: {} } as any); }); mockedApi.put.mockResolvedValue({ @@ -118,6 +135,9 @@ describe('AdminSystemPage', () => { expect(screen.getByText(/25.8 ms probe/i)).toBeTruthy(); expect(screen.getByText('OCR eng')).toBeTruthy(); expect(screen.getAllByText(/ollama configured/i).length).toBeGreaterThan(0); + expect(screen.getByText(/ollama version/i)).toBeTruthy(); + expect(screen.getByText(/model · qwen2.5:7b/i)).toBeTruthy(); + expect(screen.getByText(/cv benchmark review/i)).toBeTruthy(); expect(screen.getByText('OCR avg latency')).toBeTruthy(); expect(screen.getByText('88.4 ms')).toBeTruthy(); }); diff --git a/job-tracker-ui/src/pages/AdminSystemPage.tsx b/job-tracker-ui/src/pages/AdminSystemPage.tsx index b559315..7b43b08 100644 --- a/job-tracker-ui/src/pages/AdminSystemPage.tsx +++ b/job-tracker-ui/src/pages/AdminSystemPage.tsx @@ -26,6 +26,14 @@ type AiServiceMetrics = { gpuName?: string | null; ocrAvailable?: boolean | null; ocrLanguages?: string | null; + ollamaConfigured?: boolean | null; + ollamaReachable?: boolean | null; + ollamaModel?: string | null; + ollamaModelAvailable?: boolean | null; + ollamaVersion?: string | null; + ollamaInstalledModels?: string[] | null; + ollamaLoadedModels?: string[] | null; + ollamaLoadedCount?: number | null; healthLatencyMs?: number | null; probeLatencyMs?: number | null; lastProbeAt?: string | null; @@ -60,6 +68,13 @@ type EditableEmailSettings = { hasPassword: boolean; }; +type CvBenchmarkStatus = { + indexJson?: string | null; + reportMarkdown?: string | null; + rootPath: string; + lastUpdatedAtUtc?: string | null; +}; + type SystemStatus = { environment: string; contentRoot: string; @@ -141,6 +156,7 @@ export default function AdminSystemPage() { const [tab, setTab] = useState(0); const [status, setStatus] = useState(null); const [emailSettings, setEmailSettings] = useState(null); + const [benchmarkStatus, setBenchmarkStatus] = useState(null); const [smtpPassword, setSmtpPassword] = useState(""); const [clearPassword, setClearPassword] = useState(false); const [loading, setLoading] = useState(false); @@ -156,18 +172,21 @@ export default function AdminSystemPage() { setLoading(true); setError(null); try { - const [statusRes, emailRes] = await Promise.all([ + const [statusRes, emailRes, benchmarkRes] = await Promise.all([ api.get("/admin/system"), api.get("/admin/system/email-settings"), + api.get("/admin/system/cv-benchmark").catch(() => ({ data: null } as any)), ]); setStatus(statusRes.data); setEmailSettings(emailRes.data); + setBenchmarkStatus(benchmarkRes.data ?? null); setSmtpPassword(""); setClearPassword(false); } catch (e: any) { setError(getApiErrorMessage(e, "Failed to load system status.")); setStatus(null); setEmailSettings(null); + setBenchmarkStatus(null); } finally { setLoading(false); } @@ -367,6 +386,8 @@ export default function AdminSystemPage() { + + @@ -395,6 +416,25 @@ export default function AdminSystemPage() { + {(status?.ai.ollamaInstalledModels ?? []).slice(0, 4).map((model) => ( + + ))} + {(status?.ai.ollamaLoadedModels ?? []).slice(0, 3).map((model) => ( + + ))} + + + + + CV benchmark review + + + + + + + {benchmarkStatus?.reportMarkdown || "Run scripts/run-cv-benchmark.sh to generate the latest corpus report and fixture candidates."} + diff --git a/job-tracker-ui/src/pages/ProfilePage.tsx b/job-tracker-ui/src/pages/ProfilePage.tsx index ddf32dd..5992d35 100644 --- a/job-tracker-ui/src/pages/ProfilePage.tsx +++ b/job-tracker-ui/src/pages/ProfilePage.tsx @@ -26,7 +26,7 @@ import { JobApplication } from "../types"; type CvSectionOption = "" | "Professional Summary" | "Core Skills" | "Experience Highlights" | "Selected Achievements" | "Projects"; -type CvSectionStyle = "ats-minimal" | "harvard" | "auckland" | "edinburgh"; +type CvSectionStyle = "ats-minimal" | "harvard" | "auckland" | "edinburgh" | "monarch" | "fjord"; type ExtractionRun = { id: number; @@ -60,6 +60,18 @@ type RewriteTemplateOption = { sampleBullets: string[]; }; +type CvBuilderPreview = { + templateId: CvSectionStyle; + html: string; + suggestedFileName: string; + fullText: string; + rewrittenText: string; + structuredCv: StructuredCvProfile; + sectionName?: string | null; + targetRole?: string | null; + jobApplicationId?: number | null; +}; + type MeResponse = { provider?: "local" | "google" | "external"; id?: string; @@ -122,6 +134,26 @@ const REWRITE_TEMPLATES: RewriteTemplateOption[] = [ sampleMeta: "Premium spacing · stronger visual voice", sampleBullets: ["Useful when the CV should feel more distinctive.", "Still keeps wording grounded and factual."] }, + { + id: "monarch", + title: "Monarch", + eyebrow: "Executive", + accent: "#7c2d12", + blurb: "High-contrast premium presentation for leadership-heavy applications.", + sampleHeading: "Executive Profile", + sampleMeta: "Leadership clarity · premium hierarchy", + sampleBullets: ["Adds more top-level summary emphasis.", "Well suited to senior strategic roles."] + }, + { + id: "fjord", + title: "Fjord", + eyebrow: "Technical", + accent: "#0f4c5c", + blurb: "Calm, high-density layout for engineering resumes and project-heavy CVs.", + sampleHeading: "Projects & Systems", + sampleMeta: "Technical depth · practical readability", + sampleBullets: ["Gives projects and skills more weight.", "Better for technical detail without chaos."] + }, ]; function initialsFrom(values: Array) { @@ -208,7 +240,7 @@ export default function ProfilePage() { const [cvSectionStyle, setCvSectionStyle] = useState("ats-minimal"); const [cvSectionTargetRole, setCvSectionTargetRole] = useState(""); const [selectedRewriteJobId, setSelectedRewriteJobId] = useState(""); - const [cvSectionDraft, setCvSectionDraft] = useState(""); + const [rewritePreview, setRewritePreview] = useState(null); const [rewritePreviewTemplate, setRewritePreviewTemplate] = useState(null); const [savedJobs, setSavedJobs] = useState([]); const [parsingCvSections, setParsingCvSections] = useState(false); @@ -263,6 +295,7 @@ export default function ProfilePage() { const latestRun = extractionRuns[0]; const selectedRewriteTemplate = REWRITE_TEMPLATES.find((option) => option.id === cvSectionStyle) ?? REWRITE_TEMPLATES[0]; const selectedRewriteJob = savedJobs.find((job) => String(job.id) === selectedRewriteJobId) ?? null; + const rewriteReady = Boolean(rewritePreview?.html && rewritePreview.fullText.trim()); return ( @@ -742,39 +775,22 @@ export default function ProfilePage() { ))} - - + + - CV style rewrite studio - {t("profileCvSectionToolsHelp")} + Template-driven CV builder + + Choose a template, optionally target one section, and tailor the output toward a saved job or free-text role target. The preview below renders the actual PDF layout before you apply it. + + + + + {selectedRewriteJob ? : null} + {rewriteReady ? : null} - - + {REWRITE_TEMPLATES.map((option) => { const selected = option.id === cvSectionStyle; return ( @@ -791,25 +807,27 @@ export default function ProfilePage() { }} sx={{ p: 1.5, - borderRadius: 3, + borderRadius: 3.5, cursor: "pointer", border: "1px solid", borderColor: selected ? "primary.main" : "divider", - boxShadow: selected ? "0 0 0 1px rgba(25,118,210,0.15)" : "none", - backgroundColor: selected ? "rgba(25,118,210,0.06)" : "background.paper", + boxShadow: selected ? "0 0 0 1px rgba(25,118,210,0.18), 0 12px 30px rgba(15,23,42,0.08)" : "0 6px 18px rgba(15,23,42,0.04)", + background: selected ? `linear-gradient(180deg, ${option.accent}12 0%, rgba(255,255,255,0.96) 100%)` : "background.paper", + transition: "transform 120ms ease, box-shadow 120ms ease, border-color 120ms ease", + '&:hover': { transform: 'translateY(-2px)' }, }} > - {option.eyebrow} - {option.title} + {option.eyebrow} + {option.title} { event.stopPropagation(); setRewritePreviewTemplate(option); }}> - - {option.sampleHeading} + + {option.sampleHeading} {option.sampleMeta} {option.sampleBullets.map((bullet) => ( • {bullet} @@ -821,7 +839,7 @@ export default function ProfilePage() { })} - + {t("profileCvSectionLabel")} - setCvSectionTargetRole(e.target.value)} fullWidth helperText={selectedRewriteJob ? `Using saved job context: ${selectedRewriteJob.jobTitle}` : undefined} /> + setCvSectionTargetRole(e.target.value)} + fullWidth + helperText={selectedRewriteJob ? `Using saved job context: ${selectedRewriteJob.jobTitle}` : "Leave empty to let the selected job drive tailoring."} + /> Saved job context