diff --git a/JobTrackerApi.Tests/CvDocumentImportTests.cs b/JobTrackerApi.Tests/CvDocumentImportTests.cs index 4294515..98b1f2d 100644 --- a/JobTrackerApi.Tests/CvDocumentImportTests.cs +++ b/JobTrackerApi.Tests/CvDocumentImportTests.cs @@ -7,6 +7,60 @@ namespace JobTrackerApi.Tests; public class CvDocumentImportTests { + [Fact] + public void Grounded_duplicate_skills_keep_evidence_at_their_final_normalized_indices() + { + var candidate = new StructuredCvProfile { Skills = { "C#", "C#", "SQL" } }; + var source = Document(("C#", "paragraph"), ("SQL", "paragraph")); + var result = StructuredCvProfileJson.DeserializePersisted(StructuredCvProfileJson.SerializePersisted(CvDocumentReview.Ground(candidate, source))); + Assert.Equal(new[] { "C#", "SQL" }, result.Skills); + Assert.Equal("SQL", result.Metadata.Fields["skills.1"].SourceSnippet); + Assert.DoesNotContain("skills.2", result.Metadata.Fields.Keys); + } + [Fact] + public void Appended_bullet_metadata_follows_the_actual_destination_and_preserves_old_evidence() + { + var current = new StructuredCvProfile { Jobs = { new() { Title = "Engineer", Company = "Example", Start = "2020", End = "2021", Bullets = { "Old" } } } }; + current.Metadata.Fields["jobs.0.bullets.0"] = new() { SourceSnippet = "Old" }; + var incoming = new StructuredCvProfile { Jobs = { new() { Title = "Engineer", Company = "Example", Start = "2020", End = "2021", Bullets = { "New" } } } }; + incoming.Metadata.Fields["jobs.0.bullets.0"] = new() { SourceSnippet = "New" }; + var result = CvDocumentReview.Apply(current, incoming, new[] { new CvImportDecision("jobs.0.bullets.0", "use") }); + Assert.Equal(new[] { "Old", "New" }, result.Jobs[0].Bullets); + Assert.Equal("Old", result.Metadata.Fields["jobs.0.bullets.0"].SourceSnippet); + Assert.Equal("New", result.Metadata.Fields["jobs.0.bullets.1"].SourceSnippet); + } + + [Fact] + public void Duplicate_imported_languages_and_jobs_do_not_create_duplicate_records() + { + var incoming = new StructuredCvProfile { Languages = { new() { Name = "English", Level = "B2" }, new() { Name = "English", Level = "B2" } }, Jobs = { new() { Title = "Engineer", Company = "A", Start = "2020", End = "2021" }, new() { Title = "Engineer", Company = "A", Start = "2020", End = "2021" }, new() { Title = "Engineer", Company = "A", Start = "2022", End = "2023" } } }; + var decisions = CvDocumentReview.Build(new(), incoming).SelectMany(g => g.Fields).Select(f => new CvImportDecision(f.Id, "use")).ToArray(); + var result = CvDocumentReview.Apply(new(), incoming, decisions); + Assert.Single(result.Languages); + Assert.Equal(2, result.Jobs.Count); + } + + [Fact] + public void Correcting_a_new_skill_to_an_existing_skill_keeps_metadata_on_a_real_item() + { + var result = CvDocumentReview.Apply(new() { Skills = { "C#" } }, new() { Skills = { "SQL" } }, new[] { new CvImportDecision("skills.0", "correct", "C#") }); + var persisted = StructuredCvProfileJson.DeserializePersisted(StructuredCvProfileJson.SerializePersisted(result)); + Assert.Single(persisted.Skills); + Assert.Contains("skills.0", persisted.Metadata.Fields.Keys); + Assert.DoesNotContain("skills.1", persisted.Metadata.Fields.Keys); + } + [Fact] + public void Reviewed_metadata_retains_its_own_source_identity_across_later_imports() + { + var incoming = StructuredCvProfileJson.DeserializePersisted("""{"skills":["C#"],"metadata":{"fields":{"skills.0":{"method":"deterministic-source","sourceBlockId":"b0001","sourceArtifactId":21,"sourceExtractionRunId":7,"sourceFileSha256":"abc"}}}}"""); + var merged = CvDocumentReview.Apply(new(), incoming, new[] { new CvImportDecision("skills.0", "use") }); + using var json = System.Text.Json.JsonDocument.Parse(StructuredCvProfileJson.SerializePersisted(merged)); + var field = json.RootElement.GetProperty("metadata").GetProperty("fields").GetProperty("skills.0"); + Assert.True(field.TryGetProperty("sourceExtractionRunId", out var run)); + Assert.Equal(7, run.GetInt32()); + Assert.Equal(21, field.GetProperty("sourceArtifactId").GetInt32()); + Assert.Equal("abc", field.GetProperty("sourceFileSha256").GetString()); + } [Fact] public void Imported_contact_links_append_without_replacing_unrelated_existing_links() { diff --git a/JobTrackerApi.Tests/CvProcessingOperationTests.cs b/JobTrackerApi.Tests/CvProcessingOperationTests.cs index 621827b..f367d50 100644 --- a/JobTrackerApi.Tests/CvProcessingOperationTests.cs +++ b/JobTrackerApi.Tests/CvProcessingOperationTests.cs @@ -19,6 +19,75 @@ namespace JobTrackerApi.Tests; public sealed class CvProcessingOperationTests { + [Fact] + public async Task Worker_that_lost_its_lease_cannot_overwrite_a_recovered_successful_review() + { + await using var fixture = await Fixture.CreateAsync(); + await fixture.SeedUserAsync(); + var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + fixture.DocumentExtraction = async () => { entered.TrySetResult(); await release.Task; return new("Skills\nC#", false, "application/pdf", 1, 9, "synthetic.pdf", new[] { new CvImportBlock("b1", "Skills\nC#", "paragraph", 1) }, Array.Empty(), "docling-native-pdf-v1"); }; + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + using var owner = scope.ServiceProvider.GetRequiredService().UseBackgroundUser("user-1"); + var bytes = Encoding.UTF8.GetBytes("%PDF-1.4\nSynthetic fixture"); + Assert.IsType(await CreateController(scope.ServiceProvider).Import(new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "synthetic.pdf") { Headers = new HeaderDictionary(), ContentType = "application/pdf" })); + } + var worker = fixture.Provider.GetRequiredService().RunOnceAsync(default); + try + { + await entered.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await using var recovered = fixture.Provider.CreateAsyncScope(); + var db = recovered.ServiceProvider.GetRequiredService(); + var now = DateTime.UtcNow; + await db.UserOperations.IgnoreQueryFilters().ExecuteUpdateAsync(s => s.SetProperty(o => o.Status, OperationStatuses.Succeeded).SetProperty(o => o.LeaseToken, (string?)null).SetProperty(o => o.CompletedAtUtc, now)); + await db.CvExtractionRuns.IgnoreQueryFilters().ExecuteUpdateAsync(s => s.SetProperty(r => r.Status, "pending_review").SetProperty(r => r.StructuredProfileJson, "{\"skills\":[\"Recovered winner\"]}")); + } + finally { release.TrySetResult(); } + Assert.True(await worker.WaitAsync(TimeSpan.FromSeconds(10))); + await using var verify = fixture.Provider.CreateAsyncScope(); + var winner = await verify.ServiceProvider.GetRequiredService().CvExtractionRuns.IgnoreQueryFilters().SingleAsync(); + Assert.Equal("pending_review", winner.Status); + Assert.Contains("Recovered winner", winner.StructuredProfileJson); + } + [Fact] + public async Task Document_failure_message_is_bounded_and_does_not_crash_the_worker_or_disclose_source() + { + await using var fixture = await Fixture.CreateAsync(); + await fixture.SeedUserAsync(); + fixture.ExtractionFailure = new InvalidOperationException("PRIVATE_SOURCE " + new string('x', 2000)); + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + using var owner = scope.ServiceProvider.GetRequiredService().UseBackgroundUser("user-1"); + var bytes = Encoding.UTF8.GetBytes("%PDF-1.4\nSynthetic fixture"); + Assert.IsType(await CreateController(scope.ServiceProvider).Import(new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "synthetic.pdf") { Headers = new HeaderDictionary(), ContentType = "application/pdf" })); + } + Assert.True(await fixture.Provider.GetRequiredService().RunOnceAsync(default)); + await using var verify = fixture.Provider.CreateAsyncScope(); + var operation = await verify.ServiceProvider.GetRequiredService().UserOperations.IgnoreQueryFilters().SingleAsync(); + Assert.Equal(OperationStatuses.Failed, operation.Status); + Assert.InRange(operation.FailureMessage!.Length, 1, 512); + Assert.DoesNotContain("PRIVATE_SOURCE", operation.FailureMessage); + } + [Fact] + public async Task Structured_document_worker_publishes_review_under_its_real_database_lease() + { + await using var fixture = await Fixture.CreateAsync(); + await fixture.SeedUserAsync(); + CvProcessingOperationResponse response; + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + using var owner = scope.ServiceProvider.GetRequiredService().UseBackgroundUser("user-1"); + var bytes = Encoding.UTF8.GetBytes("%PDF-1.4\nSynthetic transport fixture"); + response = Response(await CreateController(scope.ServiceProvider).Import(new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "synthetic.pdf") { Headers = new HeaderDictionary(), ContentType = "application/pdf" })); + } + Assert.True(await fixture.Provider.GetRequiredService().RunOnceAsync(default)); + await using var verify = fixture.Provider.CreateAsyncScope(); + var db = verify.ServiceProvider.GetRequiredService(); + Assert.Equal("pending_review", (await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync()).Status); + Assert.Equal(OperationStatuses.Succeeded, (await db.UserOperations.IgnoreQueryFilters().SingleAsync()).Status); + Assert.Empty(await db.CareerProfiles.IgnoreQueryFilters().ToListAsync()); + } [Fact] public async Task Cancellation_of_an_expired_import_operation_cannot_resurrect_its_review() { @@ -227,6 +296,8 @@ public sealed class CvProcessingOperationTests private readonly string _tempRoot; public ServiceProvider Provider { get; } public AiGenerationException? GenerationFailure { get; set; } + public Exception? ExtractionFailure { get; set; } + public Func>? DocumentExtraction { get; set; } private Fixture(SqliteConnection connection, string tempRoot, ServiceProvider provider) { @@ -251,6 +322,8 @@ public sealed class CvProcessingOperationTests environment.SetupGet(item => item.ContentRootPath).Returns(tempRoot); Fixture? fixture = null; var summarizer = new Mock(); + summarizer.Setup(item => item.ExtractCvDocumentAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async () => fixture?.ExtractionFailure is not null ? throw fixture.ExtractionFailure : fixture?.DocumentExtraction is not null ? await fixture.DocumentExtraction() : new CvImportDocument("Skills\nC#", false, "application/pdf", 1, 9, "synthetic.pdf", new[] { new CvImportBlock("b1", "Skills", "section_header", 1), new CvImportBlock("b2", "C#", "paragraph", 1) }, Array.Empty(), "docling-native-pdf-v1")); summarizer.Setup(item => item.ExtractTextAsync( It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new AiTextExtractionResult( diff --git a/JobTrackerApi.Tests/ProfileCvControllerTests.cs b/JobTrackerApi.Tests/ProfileCvControllerTests.cs index dccde7a..150c0d7 100644 --- a/JobTrackerApi.Tests/ProfileCvControllerTests.cs +++ b/JobTrackerApi.Tests/ProfileCvControllerTests.cs @@ -20,6 +20,46 @@ namespace JobTrackerApi.Tests; public sealed class ProfileCvControllerTests { + [Fact] + public async Task Pending_import_review_remains_reachable_after_ten_newer_completed_runs() + { + var user = new ApplicationUser { Id = "user-1" }; + var users = CreateUserManager(); + users.Setup(x => x.GetUserAsync(It.IsAny())).ReturnsAsync(user); + await using var db = CreateDb(); + var pending = new CvExtractionRun { OwnerUserId = user.Id, Status = "pending_review", StartedAtUtc = DateTimeOffset.UtcNow.AddDays(-5), ParserVersion = "docling-v1" }; + db.CvExtractionRuns.Add(pending); + db.CvExtractionRuns.AddRange(Enumerable.Range(0, 12).Select(i => new CvExtractionRun { OwnerUserId = user.Id, Status = "applied", StartedAtUtc = DateTimeOffset.UtcNow.AddMinutes(-i) })); + await db.SaveChangesAsync(); + var result = await CreateController(users.Object, Mock.Of(), db, CreatePaths()).GetRuns(); + var runs = Assert.IsAssignableFrom>(Assert.IsType(result.Result).Value).ToList(); + Assert.Contains(runs, r => r.Id == pending.Id); + Assert.Equal(11, runs.Count); + } + [Fact] + public async Task Document_import_rejects_docx_dtd_before_admission_or_file_storage() + { + var user = new ApplicationUser { Id = "user-1" }; + var users = CreateUserManager(); + users.Setup(x => x.GetUserAsync(It.IsAny())).ReturnsAsync(user); + await using var db = CreateDb(); + var paths = CreatePaths(); + var controller = CreateController(users.Object, Mock.Of(), db, paths); + using var bytes = new MemoryStream(); + using (var archive = new System.IO.Compression.ZipArchive(bytes, System.IO.Compression.ZipArchiveMode.Create, true)) + { + foreach (var entry in new Dictionary { ["[Content_Types].xml"] = "", ["word/document.xml"] = "", ["word/_rels/document.xml.rels"] = "]>&example;" }) + { + await using var writer = new StreamWriter(archive.CreateEntry(entry.Key).Open()); + await writer.WriteAsync(entry.Value); + } + } + bytes.Position = 0; + var file = new FormFile(bytes, 0, bytes.Length, "file", "synthetic.docx") { Headers = new HeaderDictionary(), ContentType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document" }; + Assert.IsType(await controller.Import(file)); + Assert.Empty(db.CvUploadArtifacts); + Assert.Empty(Directory.EnumerateFiles(paths.CvArtifactsRoot, "*", SearchOption.AllDirectories)); + } [Fact] public async Task Document_import_rejects_spoofed_pdf_without_storing_it() { diff --git a/JobTrackerApi/Controllers/ProfileCvController.Import.cs b/JobTrackerApi/Controllers/ProfileCvController.Import.cs index bb726ad..1624c6a 100644 --- a/JobTrackerApi/Controllers/ProfileCvController.Import.cs +++ b/JobTrackerApi/Controllers/ProfileCvController.Import.cs @@ -42,11 +42,17 @@ public sealed partial class ProfileCvController var candidate = CvDocumentInterpreter.Parse(blocks); await Stage("cv_validating_details"); var profile = CvDocumentReview.Ground(candidate, document); + foreach (var field in profile.Metadata.Fields.Values) + { + field.SourceArtifactId = artifact.Id; + field.SourceExtractionRunId = run.Id; + field.SourceFileSha256 = artifact.Sha256; + } await Stage("cv_preparing_review"); await Stage("cv_review_ready"); cancellationToken.ThrowIfCancellationRequested(); var documentJson = JsonSerializer.Serialize(document); - var profileJson = StructuredCvProfileJson.Serialize(profile); + var profileJson = StructuredCvProfileJson.SerializePersisted(profile); if (context is null) { run.DocumentJson = documentJson; @@ -54,8 +60,9 @@ public sealed partial class ProfileCvController return; } var currentLease = context.Lease; + var completedAt = DateTimeOffset.UtcNow; var published = await _db.CvExtractionRuns.Where(r => r.Id == run.Id && r.OwnerUserId == run.OwnerUserId && r.Status == "running" && _db.UserOperations.Any(o => o.Id == currentLease.OperationId && o.LeaseToken == currentLease.LeaseToken && o.Status == OperationStatuses.Running && o.CancellationRequestedAtUtc == null && o.LeaseExpiresAtUtc > DateTime.UtcNow)) - .ExecuteUpdateAsync(s => s.SetProperty(r => r.DocumentJson, documentJson).SetProperty(r => r.RawExtractedText, document.Text).SetProperty(r => r.NormalizedText, document.Text).SetProperty(r => r.StructuredProfileJson, profileJson).SetProperty(r => r.Status, "pending_review").SetProperty(r => r.CompletedAtUtc, DateTimeOffset.UtcNow), cancellationToken); + .ExecuteUpdateAsync(s => s.SetProperty(r => r.DocumentJson, documentJson).SetProperty(r => r.RawExtractedText, document.Text).SetProperty(r => r.NormalizedText, document.Text).SetProperty(r => r.StructuredProfileJson, profileJson).SetProperty(r => r.Status, "pending_review").SetProperty(r => r.CompletedAtUtc, completedAt), cancellationToken); if (published != 1) throw new OperationCanceledException("The document processing lease was lost before publication.", cancellationToken); await _db.Entry(run).ReloadAsync(cancellationToken); await PruneExtractionRunsAsync(run.OwnerUserId, cancellationToken); @@ -126,7 +133,8 @@ public sealed partial class ProfileCvController using var reader = new StreamReader(entry.Open()); var relationships = await reader.ReadToEndAsync(cancellationToken); // Hyperlinks remain inert text; linked templates/OLE/media are refused. - var xml = System.Xml.Linq.XDocument.Parse(relationships); + using var xmlReader = System.Xml.XmlReader.Create(new StringReader(relationships), new System.Xml.XmlReaderSettings { DtdProcessing = System.Xml.DtdProcessing.Prohibit, XmlResolver = null, MaxCharactersInDocument = 2 * 1024 * 1024 }); + var xml = System.Xml.Linq.XDocument.Load(xmlReader); if (xml.Descendants().Any(e => string.Equals((string?)e.Attribute("TargetMode"), "External", StringComparison.OrdinalIgnoreCase) && !((string?)e.Attribute("Type") ?? "").EndsWith("/hyperlink", StringComparison.Ordinal))) throw new InvalidDataException(); } } @@ -151,7 +159,7 @@ public sealed partial class ProfileCvController if (run.ParserVersion != DocumentParserVersion || string.IsNullOrWhiteSpace(run.StructuredProfileJson)) return Conflict("This import is not ready for review."); var snapshot = await _careerProfileService.LoadSnapshotAsync(user.Id, HttpContext.RequestAborted); var document = JsonSerializer.Deserialize(run.DocumentJson ?? "null"); - return Ok(new CvImportReview(run.Id, run.Status, snapshot.Revision, run.Artifact?.OriginalFileName, $"/api/profile-cv/runs/{run.Id}/source", run.RawExtractedText, document?.Warnings ?? Array.Empty(), CvDocumentReview.Build(snapshot.Profile, StructuredCvProfileJson.Deserialize(run.StructuredProfileJson)))); + return Ok(new CvImportReview(run.Id, run.Status, snapshot.Revision, run.Artifact?.OriginalFileName, $"/api/profile-cv/runs/{run.Id}/source", run.RawExtractedText, document?.Warnings ?? Array.Empty(), CvDocumentReview.Build(snapshot.Profile, StructuredCvProfileJson.DeserializePersisted(run.StructuredProfileJson)))); } [HttpGet("runs/{id:int}/source")] @@ -189,11 +197,11 @@ public sealed partial class ProfileCvController if (run.Status != "pending_review" || string.IsNullOrWhiteSpace(run.StructuredProfileJson)) return Conflict("This import is not awaiting review."); var snapshot = await _careerProfileService.LoadSnapshotAsync(user.Id, HttpContext.RequestAborted); if (snapshot.Revision != request.ExpectedVersion) throw new DbUpdateConcurrencyException(); - var merged = CvDocumentReview.Apply(snapshot.Profile, StructuredCvProfileJson.Deserialize(run.StructuredProfileJson), request.Decisions); + var merged = CvDocumentReview.Apply(snapshot.Profile, StructuredCvProfileJson.DeserializePersisted(run.StructuredProfileJson), request.Decisions); if (!request.Decisions.Any(d => d.Action != "keep")) return BadRequest("No imported fields were selected. Discard this import to keep your current profile."); var validation = CareerProfileValidator.Validate(merged); if (validation is not null || StructuredCvProfileJson.SerializePersisted(merged).Length > 500000) return BadRequest(validation ?? "The reviewed profile exceeds its size limit."); - merged.Metadata.ProfileVersion = (user.CurrentCvProfileVersion ?? 0) + 1; + merged.Metadata.ProfileVersion = snapshot.Revision + 1; merged.Metadata.AppliedExtractionRunId = run.Id; merged.Metadata.UpdatedAtUtc = DateTimeOffset.UtcNow; await _careerProfileService.SaveVersionAsync(user.Id, merged, "document_import:reviewed", request.ExpectedVersion.Value, HttpContext.RequestAborted); diff --git a/JobTrackerApi/Controllers/ProfileCvController.Pipeline.cs b/JobTrackerApi/Controllers/ProfileCvController.Pipeline.cs index e4f7bcc..8304dab 100644 --- a/JobTrackerApi/Controllers/ProfileCvController.Pipeline.cs +++ b/JobTrackerApi/Controllers/ProfileCvController.Pipeline.cs @@ -411,9 +411,19 @@ public sealed partial class ProfileCvController : ControllerBase ? await _db.CvExtractionRuns.IgnoreQueryFilters().FirstOrDefaultAsync(x => x.Id == runId, cancellationToken) : await _db.CvExtractionRuns.FirstOrDefaultAsync(x => x.Id == runId && x.OwnerUserId == ownerUserId, cancellationToken); if (run is null) return null; + var guardedDocument = run.ParserVersion == DocumentParserVersion && operationContext is not null; + if (guardedDocument) + { + var lease = operationContext!.Lease; + var claimed = await _db.CvExtractionRuns.Where(r => r.Id == run.Id && r.OwnerUserId == run.OwnerUserId && (r.Status == "queued" || r.Status == "running") && _db.UserOperations.Any(o => o.Id == lease.OperationId && o.LeaseToken == lease.LeaseToken && o.Status == OperationStatuses.Running && o.CancellationRequestedAtUtc == null && o.LeaseExpiresAtUtc > DateTime.UtcNow)) + .ExecuteUpdateAsync(s => s.SetProperty(r => r.Status, "running").SetProperty(r => r.ErrorMessage, (string?)null), cancellationToken); + if (claimed != 1) { _db.Entry(run).State = EntityState.Detached; throw new OperationCanceledException("The import worker lease is no longer active."); } + await _db.Entry(run).ReloadAsync(cancellationToken); + } var user = await _users.FindByIdAsync(run.OwnerUserId); if (user is null) { + if (guardedDocument) throw new AiOperationFailure("cv_user_not_found", "The import account is no longer available.", false); run.Status = "failed"; run.ErrorMessage = "CV processing user was not found."; run.CompletedAtUtc = DateTimeOffset.UtcNow; @@ -423,6 +433,7 @@ public sealed partial class ProfileCvController : ControllerBase if (!user.AiEnabled || !AccountPlans.ForRoles(await _users.GetRolesAsync(user)).Ai) { + if (guardedDocument) throw new AiOperationFailure("entitlement_changed", "CV import access changed before processing began.", false); run.Status = "failed"; run.ErrorMessage = user.AiEnabled ? "This AI feature requires Pro." @@ -432,9 +443,12 @@ public sealed partial class ProfileCvController : ControllerBase return new CvProcessingOutcome(false, "entitlement_changed", run.ErrorMessage); } - run.Status = "running"; - run.ErrorMessage = null; - await _db.SaveChangesAsync(cancellationToken); + if (!guardedDocument) + { + run.Status = "running"; + run.ErrorMessage = null; + await _db.SaveChangesAsync(cancellationToken); + } try { @@ -535,20 +549,36 @@ public sealed partial class ProfileCvController : ControllerBase } var generationFailure = ex as AiGenerationException; var retryable = generationFailure?.Retryable == true; - var failureMessage = generationFailure?.Message + var failureMessage = run.ParserVersion == DocumentParserVersion + ? "The document could not be processed safely. Retry or upload a supported PDF or DOCX CV." + : generationFailure?.Message ?? (ex is InvalidOperationException ? ex.Message : "CV processing failed unexpectedly. Please try again."); - run.Status = retryable ? "queued" : "failed"; - run.ErrorMessage = failureMessage; - run.CompletedAtUtc = retryable ? null : DateTimeOffset.UtcNow; - await _db.SaveChangesAsync(cancellationToken); + if (guardedDocument) + { + var lease = operationContext!.Lease; + var failedAt = retryable ? (DateTimeOffset?)null : DateTimeOffset.UtcNow; + var failedStatus = retryable ? "queued" : "failed"; + var updated = await _db.CvExtractionRuns.Where(r => r.Id == run.Id && (r.Status == "queued" || r.Status == "running") && _db.UserOperations.Any(o => o.Id == lease.OperationId && o.LeaseToken == lease.LeaseToken && o.Status == OperationStatuses.Running && o.CancellationRequestedAtUtc == null && o.LeaseExpiresAtUtc > DateTime.UtcNow)) + .ExecuteUpdateAsync(s => s.SetProperty(r => r.Status, failedStatus).SetProperty(r => r.ErrorMessage, failureMessage).SetProperty(r => r.CompletedAtUtc, failedAt), cancellationToken); + _db.Entry(run).State = EntityState.Detached; + if (updated != 1) throw new OperationCanceledException("The import worker lease is no longer active."); + } + else + { + run.Status = retryable ? "queued" : "failed"; + run.ErrorMessage = failureMessage; + run.CompletedAtUtc = retryable ? null : DateTimeOffset.UtcNow; + await _db.SaveChangesAsync(cancellationToken); + } if (!retryable) { await PruneExtractionRunsAsync(user.Id, cancellationToken); await SendRunCompletionEmailAsync(user, run, false, cancellationToken); } - _logger.LogWarning(ex, "CV processing run {RunId} failed for user {UserId}", run.Id, user.Id); + if (run.ParserVersion == DocumentParserVersion) _logger.LogWarning("CV document processing run {RunId} failed ({Category}).", run.Id, generationFailure?.Category ?? "cv_processing_failed"); + else _logger.LogWarning(ex, "CV processing run {RunId} failed for user {UserId}", run.Id, user.Id); return new CvProcessingOutcome( false, generationFailure?.Category ?? "cv_processing_failed", @@ -564,7 +594,7 @@ public sealed partial class ProfileCvController : ControllerBase { run.RawExtractedText = rawText; run.NormalizedText = normalizedText; - run.StructuredProfileJson = StructuredCvProfileJson.Serialize(structuredCv); + run.StructuredProfileJson = run.ParserVersion == DocumentParserVersion ? StructuredCvProfileJson.SerializePersisted(structuredCv) : StructuredCvProfileJson.Serialize(structuredCv); run.Status = "pending_review"; run.CompletedAtUtc = DateTimeOffset.UtcNow; await _db.SaveChangesAsync(cancellationToken); diff --git a/JobTrackerApi/Controllers/ProfileCvController.cs b/JobTrackerApi/Controllers/ProfileCvController.cs index b57ca50..c540a62 100644 --- a/JobTrackerApi/Controllers/ProfileCvController.cs +++ b/JobTrackerApi/Controllers/ProfileCvController.cs @@ -219,7 +219,8 @@ public sealed partial class ProfileCvController : ControllerBase .Where(x => x.OwnerUserId == user.Id); var isSqlite = _db.Database.IsSqlite(); if (!isSqlite) - runsQuery = runsQuery.OrderByDescending(x => x.StartedAtUtc).Take(10); + runsQuery = runsQuery.OrderByDescending(x => x.StartedAtUtc).Take(10) + .Union(_db.CvExtractionRuns.AsNoTracking().Where(x => x.OwnerUserId == user.Id && x.Status == "pending_review")); var runs = await runsQuery.Select(x => new CvExtractionRunListItem( x.Id, x.Trigger, @@ -233,8 +234,8 @@ public sealed partial class ProfileCvController : ControllerBase x.LlmPromptVersion, x.ErrorMessage, null)).ToListAsync(HttpContext.RequestAborted); - if (isSqlite) - runs = runs.OrderByDescending(x => x.StartedAtUtc).Take(10).ToList(); + runs = runs.OrderByDescending(x => x.StartedAtUtc).Take(10).Concat(runs.Where(x => x.Status == "pending_review")) + .DistinctBy(x => x.Id).OrderByDescending(x => x.StartedAtUtc).ToList(); var runIds = runs.Select(run => run.Id.ToString(System.Globalization.CultureInfo.InvariantCulture)).ToList(); var operations = await _db.UserOperations.AsNoTracking() diff --git a/JobTrackerApi/Models/StructuredCvProfile.cs b/JobTrackerApi/Models/StructuredCvProfile.cs index 2c67c2a..df78349 100644 --- a/JobTrackerApi/Models/StructuredCvProfile.cs +++ b/JobTrackerApi/Models/StructuredCvProfile.cs @@ -36,6 +36,9 @@ public sealed class StructuredCvFieldMetadata public string? SourceSnippet { get; set; } public int? SourcePage { get; set; } public string? SourceBlockId { get; set; } + public int? SourceArtifactId { get; set; } + public int? SourceExtractionRunId { get; set; } + public string? SourceFileSha256 { get; set; } public string? ReviewState { get; set; } public DateTimeOffset? LastUpdatedAtUtc { get; set; } } diff --git a/JobTrackerApi/Services/CvDocumentReview.cs b/JobTrackerApi/Services/CvDocumentReview.cs index 4471b5f..a3a4c7c 100644 --- a/JobTrackerApi/Services/CvDocumentReview.cs +++ b/JobTrackerApi/Services/CvDocumentReview.cs @@ -22,7 +22,7 @@ public static class CvDocumentReview public static StructuredCvProfile Ground(StructuredCvProfile candidate, CvImportDocument document) { - var node = Node(candidate); + var node = Node(StructuredCvProfileJson.NormalizeForPersistence(candidate)); node["sections"] = new JsonArray(); var metadata = new Dictionary(); void Walk(JsonNode? value, string path) @@ -162,6 +162,7 @@ public static class CvDocumentReview var selected = decisions.Where(d => d.Action != "keep").ToDictionary(d => d.Id, StringComparer.Ordinal); var result = Node(current); var linkTargets = new Dictionary(); + var newTargets = new Dictionary(StringComparer.Ordinal); foreach (var map in maps) { var choices = map.Review.Fields.Where(f => selected.ContainsKey(f.Id)).ToList(); @@ -171,8 +172,14 @@ public static class CvDocumentReview if (map.IsArray) { var list = result[map.Category]!.AsArray(); - var index = map.ExistingIndex ?? list.Count; - if (!map.ExistingIndex.HasValue) list.Add(map.Incoming is JsonObject ? new JsonObject() : JsonValue.Create("")); + var identity = map.Category + "|" + map.Incoming.ToJsonString(); + var scalarText = selected[choices[0].Id].Action == "correct" ? selected[choices[0].Id].Value! : choices[0].ExtractedValue; + var matches = Enumerable.Range(0, list.Count).Where(i => map.Incoming is JsonValue + ? string.Equals(Clean(list[i]?.ToString() ?? ""), Clean(scalarText), StringComparison.OrdinalIgnoreCase) + : Same(map.Category, list[i], map.Incoming)).ToList(); + var index = map.ExistingIndex ?? (newTargets.TryGetValue(identity, out var prior) ? prior : matches.Count == 1 ? matches[0] : list.Count); + if (index == list.Count) list.Add(map.Incoming is JsonObject ? new JsonObject() : JsonValue.Create("")); + newTargets[identity] = index; target = list[index]!; targetPrefix = map.Category + "." + index; } @@ -205,17 +212,21 @@ public static class CvDocumentReview var list = result[map.Category]!.AsArray(); list[int.Parse(targetPrefix.Split('.').Last())] = value; } - else Set(target, suffix.Split('.'), value); + else suffix = Set(target, suffix.Split('.'), value); var path = targetPrefix + (suffix.Length == 0 ? "" : "." + suffix); incoming.Metadata.Fields.TryGetValue(field.Id, out var source); var metadataNode = result["metadata"]!["fields"]!.AsObject(); - metadataNode[path] = JsonSerializer.SerializeToNode(new StructuredCvFieldMetadata { Method = decision.Action == "correct" ? "user-correction" : source?.Method ?? "deterministic-source", SourceSnippet = source?.SourceSnippet, SourcePage = source?.SourcePage, SourceBlockId = source?.SourceBlockId, ReviewState = decision.Action == "correct" ? "corrected" : "accepted", LastUpdatedAtUtc = DateTimeOffset.UtcNow }, Json); + metadataNode[path] = JsonSerializer.SerializeToNode(new StructuredCvFieldMetadata { Method = decision.Action == "correct" ? "user-correction" : source?.Method ?? "deterministic-source", SourceSnippet = source?.SourceSnippet, SourcePage = source?.SourcePage, SourceBlockId = source?.SourceBlockId, SourceArtifactId = source?.SourceArtifactId, SourceExtractionRunId = source?.SourceExtractionRunId, SourceFileSha256 = source?.SourceFileSha256, ReviewState = decision.Action == "correct" ? "corrected" : "accepted", LastUpdatedAtUtc = DateTimeOffset.UtcNow }, Json); } } - return result.Deserialize(Json)!; + var profile = result.Deserialize(Json)!; + var linkCount = profile.Contact.Links.Count; + profile = StructuredCvProfileJson.NormalizeForPersistence(profile); + if (profile.Contact.Links.Count != linkCount) throw new ArgumentException("A reviewed contact link is incomplete, duplicated or exceeds the supported limit. Keep the existing link or correct its URL."); + return profile; } - private static void Set(JsonNode target, string[] path, JsonNode value) + private static string Set(JsonNode target, string[] path, JsonNode value) { for (var i = 0; i < path.Length - 1; i++) { @@ -232,10 +243,12 @@ public static class CvDocumentReview } if (target is JsonArray array) { - var index = int.Parse(path[^1]); // Individual approved list entries append instead of erasing user-only entries. - if (!array.Any(v => v?.ToString() == value.ToString())) array.Add(value); + var index = Enumerable.Range(0, array.Count).FirstOrDefault(i => string.Equals(array[i]?.ToString(), value.ToString(), StringComparison.OrdinalIgnoreCase), -1); + if (index < 0) { index = array.Count; array.Add(value); } + path[^1] = index.ToString(System.Globalization.CultureInfo.InvariantCulture); } else target[path[^1]] = value; + return string.Join('.', path); } } diff --git a/JobTrackerApi/Services/CvProcessingQueue.cs b/JobTrackerApi/Services/CvProcessingQueue.cs index 49f3589..094c49f 100644 --- a/JobTrackerApi/Services/CvProcessingQueue.cs +++ b/JobTrackerApi/Services/CvProcessingQueue.cs @@ -74,7 +74,7 @@ public sealed class CvProcessingOperationHandler : IAiOperationHandler services, runId, operation?.CancellationRequestedAtUtc is null ? "queued" : "cancelled", - operation?.CancellationRequestedAtUtc is null ? "CV processing timed out and may be retried." : "CV processing was cancelled."); + operation?.CancellationRequestedAtUtc is null ? "CV processing timed out and may be retried." : "CV processing was cancelled.", context.Lease); throw; } if (outcome is null) @@ -88,7 +88,7 @@ public sealed class CvProcessingOperationHandler : IAiOperationHandler var canRetry = operation is not null && operation.AttemptCount < operation.MaxAttempts && (operation.DeadlineAtUtc is null || operation.DeadlineAtUtc > DateTime.UtcNow); if (!canRetry) - await SetRunStatusAsync(services, runId, "failed", outcome.FailureMessage ?? "CV processing failed."); + await SetRunStatusAsync(services, runId, "failed", outcome.FailureMessage ?? "CV processing failed.", context.Lease); } if (outcome.Provider is not null || outcome.Model is not null || outcome.RouteReason is not null) @@ -108,18 +108,20 @@ public sealed class CvProcessingOperationHandler : IAiOperationHandler outcome.Retryable); } + var isDocumentImport = await services.GetRequiredService().CvExtractionRuns.Where(r => r.Id == runId).Select(r => r.ParserVersion).FirstAsync(cancellationToken) == ProfileCvController.DocumentParserVersion; return new AiOperationExecutionResult( - $"/api/profile-cv/runs/{runId}/{(await services.GetRequiredService().CvExtractionRuns.Where(r => r.Id == runId).Select(r => r.ParserVersion).FirstAsync(cancellationToken) == ProfileCvController.DocumentParserVersion ? "review" : "diff")}", + $"/api/profile-cv/runs/{runId}/{(isDocumentImport ? "review" : "diff")}", outcome.Provider, outcome.Model, - outcome.RouteReason ?? "cv_pipeline"); + outcome.RouteReason ?? (isDocumentImport ? "cv_review_ready" : "cv_pipeline")); } - private static Task SetRunStatusAsync(IServiceProvider services, int runId, string status, string message) + private static Task SetRunStatusAsync(IServiceProvider services, int runId, string status, string message, UserOperationLease lease) { var completedAtUtc = status == "failed" || status == "cancelled" ? DateTimeOffset.UtcNow : (DateTimeOffset?)null; - return services.GetRequiredService().CvExtractionRuns - .Where(run => run.Id == runId && run.Status != "expired" && run.Status != "applied") + var db = services.GetRequiredService(); + return db.CvExtractionRuns + .Where(run => run.Id == runId && (run.Status == "running" || run.Status == "queued") && db.UserOperations.Any(o => o.Id == lease.OperationId && o.LeaseToken == lease.LeaseToken && o.Status == OperationStatuses.Running)) .ExecuteUpdateAsync(setters => setters .SetProperty(run => run.Status, status) .SetProperty(run => run.ErrorMessage, message) diff --git a/docs/cv-import/REVIEW.md b/docs/cv-import/REVIEW.md new file mode 100644 index 0000000..589435c --- /dev/null +++ b/docs/cv-import/REVIEW.md @@ -0,0 +1,13 @@ +# Review, corrections and merge qualification + +26 October 2026. The existing Career Profile page now routes PDF/DOCX through the document import pipeline. Legacy text/image upload and historical reviews remain available. Upload does not change the canonical profile. + +Desktop presents an authenticated original PDF beside grouped fields; mobile offers source/review tabs. DOCX provides ordered extracted source and the original download rather than claiming a browser-rendered pagination preview. Source evidence shows the supporting text, converter method and page where known. Uncertain relationships and converter warnings remain visible. No field starts approved. + +Users can keep the current value, accept/add the imported value, correct it, or skip it. A group approval is an explicit user decision. The final Apply action submits only selected decisions and the canonical revision loaded for comparison. A conflict refresh preserves corrections but resets confirmations against the new comparison. An all-skipped import must be discarded; it cannot display a misleading saved acknowledgement. Successful application links to Career Profile and the existing CV Editor. + +Review matches records conservatively using complete identity/date keys; incomplete and multiple matches require a new record or manual correction. Existing nonmatching fields, list entries, links and CV-specific overrides remain intact. Nested list provenance follows the actual appended/deduplicated destination index. Each imported fact retains artifact/run/file-hash identity even after later imports or client saves. Grounding occurs after canonical normalization so duplicate source skills cannot shift evidence onto another value. + +Independent code review identified stale-worker publication, nested merge provenance, duplicate targets, old pending-review visibility and benchmark record association defects. Regressions reproduced them before fixes. Additional tests cover DOCX DTD rejection, inert document hyperlinks, bounded errors, per-field source identity and canonical rather than legacy projection revision numbering. Failed attempts are retained under `.artifacts/cv-import/`; focused suites overlap and are not added to final totals. + +The dedicated real import browser lane uses the actual isolated FastAPI/Docling child, durable API worker, reviewed transactional save, unchanged pre-existing CV variant, redesigned editor and actual PDF export. It exercises English/light/desktop PDF and Bokmål/dark/mobile DOCX, source ownership, refresh, no implicit save, corrections and automated accessibility. Browser automation does not qualify physical devices, human assistive technology, translation quality or printing. diff --git a/docs/cv-import/TODO.md b/docs/cv-import/TODO.md index 9f719d5..c9409c3 100644 --- a/docs/cv-import/TODO.md +++ b/docs/cv-import/TODO.md @@ -4,7 +4,7 @@ - [x] CV-IMPORT-01: secure upload and real Docling extraction (native PDF layout/OCR limitations explicit). - [x] CV-IMPORT-02: grounded structured interpretation and disabled local AI qualification seam. - [x] CV-IMPORT-03: genuine progress, lease/cancellation recovery and bounded retention; full final freeze remains separate. -- [ ] CV-IMPORT-04: review/corrections/conflicts/source UI. +- [x] CV-IMPORT-04: review/corrections/conflicts/source UI; final integrated regression tracked separately. - [ ] CV-IMPORT-05: revisioned transactional Career Profile persistence. - [ ] CV-IMPORT-06: accuracy/security/resilience qualification. - [ ] CV-IMPORT-07: full regression, acceptance and CV-QA-01 handoff. @@ -16,3 +16,5 @@ Known open acceptance gates inherited from CV-UX: live model BLOCKED; human acce Extraction subcheckpoint pushed `3ea8ff9`. Backend reviewed integration focused 51 PASS and actual provider 69 PASS; qualification remains conditional until the final frozen-source lanes. Root continued implementation after delegated workers exhausted their service usage; unfinished worker edits were inspected and tested, not assumed complete. CV-IMPORT-03 evidence: `lifecycle-green-01` 52 focused backend PASS; `backend-full-01` 1,250 PASS, zero skips. Expired-operation cancellation regression failed before the shared status guards. Linux real sidecar API attempt 03 passed PDF/DOCX and refused the scan in network-disabled/read-only isolation. Earlier attempts remain failed: Linux platform metadata subprocess and missing private HOME were fixed at the child boundary. Source retention includes orphan generated upload files and preserves unrelated files. Original source inspection is unavailable after expiry; approved canonical history is retained. + +CV-IMPORT-04: full frontend attempt 03 passed 990 tests / 108 suites, zero skips; TypeScript/lint passed. Current offline sidecar partitions passed 76 + 14 disjoint tests; focused review/source/lease regressions passed. Real import browser attempt 04 passed 2/2 before final review corrections; fresh current-source integrated rerun remains required. See REVIEW.md and BACKEND.md for retained failure corrections. diff --git a/job-tracker-ui/import-tests/cv-import-docling.spec.ts b/job-tracker-ui/import-tests/cv-import-docling.spec.ts new file mode 100644 index 0000000..a6ece66 --- /dev/null +++ b/job-tracker-ui/import-tests/cv-import-docling.spec.ts @@ -0,0 +1,80 @@ +import { expect, test, type APIRequestContext } from '@playwright/test'; +import AxeBuilder from '@axe-core/playwright'; +import path from 'node:path'; +import { translations } from '../src/i18n/translations'; + +const url = 'http://localhost:5302/api'; +const password = 'E2ePassword123!'; +async function login(api: APIRequestContext, email: string) { + expect((await api.post(`${url}/auth/login`, { data: { email, password } })).status()).toBe(200); + const me = await (await api.get(`${url}/auth/me`)).json(); + return { 'X-CSRF-TOKEN': (await api.storageState()).cookies.find(c => c.name === 'XSRF-TOKEN')!.value, 'X-Session-Context': me.sessionContext as string }; +} + +for (const scenario of [ + { language: 'en', mode: 'light', width: 1440, file: 'dev-en-simple.pdf' }, + { language: 'nb', mode: 'dark', width: 390, file: 'dev-nb-education.docx' }, +] as const) { + test(`real Docling ${scenario.language}/${scenario.mode}: upload, refresh, review, correct, durable profile, editor and PDF`, async ({ page, context, playwright }, info) => { + const t = translations[scenario.language]; + const admin = await playwright.request.newContext(); + const adminHeaders = await login(admin, 'e2e@example.test'); + const email = `cv-import-${Date.now()}-${scenario.language}@example.test`; + expect((await admin.post(`${url}/users`, { headers: adminHeaders, data: { email, password, roles: ['Premium'] } })).status()).toBe(200); + await admin.dispose(); + const headers = await login(page.request, email); + expect((await page.request.put(`${url}/auth/preferences/language`, { headers, data: { language: scenario.language === 'nb' ? 'nb-NO' : 'en' } })).status()).toBe(204); + await context.addInitScript(({ language, mode }) => { localStorage.setItem('uiLanguage', language); localStorage.setItem('jobtracker.themeMode', mode); }, scenario); + await page.setViewportSize({ width: scenario.width, height: 900 }); + const created = await page.request.post(`${url}/cv/variants`, { headers, data: { name: 'Preserved import CV', settings: { themeId: 'minimal' } } }); + expect(created.status()).toBe(200); + const cv = await created.json(); + const originalVariant = await (await page.request.get(`${url}/cv/variants/${cv.id}`)).json(); + await page.goto('/career?section=import'); + await expect(page.getByRole('button', { name: t.profileUploadCv, exact: true })).toBeEnabled(); + await page.locator('input[type=file][accept*=".pdf"]').setInputFiles(path.resolve(process.env.CV_IMPORT_FIXTURE_ROOT || '../.artifacts/cv-import/corpus-02', scenario.file)); + await expect.poll(async () => (await (await page.request.get(`${url}/profile-cv/runs`)).json())[0]?.status, { timeout: 90_000 }).toBe('pending_review'); + expect((await (await page.request.get(`${url}/career/profile`)).json()).revision).toBe(0); + const run = (await (await page.request.get(`${url}/profile-cv/runs`)).json())[0]; + expect(run.parserVersion).toBe('docling-v1'); + expect(run.operation.progressStage).toBe('cv_review_ready'); + await page.reload(); + const review = page.getByRole('region', { name: t.cvImportReviewPanel, exact: true }); + await expect(review.getByRole('button', { name: t.cvImportApplyReviewed })).toBeDisabled(); + const source = await page.request.get(`${url}/profile-cv/runs/${run.id}/source`); + expect(source.status()).toBe(200); + expect(source.headers()['cache-control']).toContain('no-store'); + const stranger = await playwright.request.newContext(); + await login(stranger, 'e2e@example.test'); + expect((await stranger.get(`${url}/profile-cv/runs/${run.id}/source`)).status()).toBe(404); + expect((await stranger.get(`${url}/profile-cv/runs/${run.id}/review`)).status()).toBe(404); + await stranger.dispose(); + const groups = review.getByRole('button', { name: t.cvImportUseGroup, exact: true }); + await expect(groups.first()).toBeVisible(); + for (const button of await groups.all()) await button.click(); + const nameChoice = review.getByRole('combobox', { name: t.cvImportDecision.replace('{field}', t.profileCvContactFullName), exact: true }); + await nameChoice.selectOption('correct'); + await review.getByRole('textbox', { name: t.cvImportCorrection.replace('{field}', t.profileCvContactFullName), exact: true }).fill('Reviewed Åse Example'); + const axe = await new AxeBuilder({ page }).withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa']).analyze(); + await info.attach('review-axe', { body: JSON.stringify({ violations: axe.violations, incomplete: axe.incomplete }), contentType: 'application/json' }); + expect(axe.violations).toEqual([]); + await page.screenshot({ path: info.outputPath('review.png'), fullPage: true }); + await review.getByRole('button', { name: t.cvImportApplyReviewed, exact: true }).click(); + await expect(page.getByText(t.cvImportSaved, { exact: true })).toBeVisible(); + const profile = await (await page.request.get(`${url}/career/profile`)).json(); + expect(profile.revision).toBe(1); + expect(profile.profile.contact.fullName).toBe('Reviewed Åse Example'); + expect((await (await page.request.get(`${url}/career/profile/versions`)).json())).toHaveLength(1); + expect(await (await page.request.get(`${url}/cv/variants/${cv.id}`)).json()).toEqual(originalVariant); + await page.goto(`/career/builder/${cv.id}`); + await expect(page.getByLabel(t.cvEditorName, { exact: true })).toHaveValue('Preserved import CV'); + if (scenario.width < 768) await page.getByRole('button', { name: t.cvEditorPreview, exact: true }).click(); + await expect(page.getByTitle(t.cvEditorPreviewPage.replace('{page}', '1'), { exact: true })).toBeVisible(); + const download = page.waitForEvent('download'); + await page.getByRole('button', { name: t.cvEditorDownloadPdf, exact: true }).click(); + const file = await download; + expect(file.suggestedFilename()).toMatch(/\.pdf$/); + await file.saveAs(info.outputPath(`imported-${scenario.language}.pdf`)); + await expect.poll(async () => (await (await page.request.get(`${url}/career/profile`)).json()).revision).toBe(1); + }); +} diff --git a/job-tracker-ui/playwright.cv-import.config.ts b/job-tracker-ui/playwright.cv-import.config.ts new file mode 100644 index 0000000..b736063 --- /dev/null +++ b/job-tracker-ui/playwright.cv-import.config.ts @@ -0,0 +1,13 @@ +import { defineConfig } from '@playwright/test'; +import ordinary from './playwright.config'; + +if (!process.env.CV_IMPORT_SIDECAR_URL) throw new Error('Set CV_IMPORT_SIDECAR_URL to the isolated local Docling service.'); +const servers = Array.isArray(ordinary.webServer) ? ordinary.webServer : [ordinary.webServer!]; +export default defineConfig({ + ...ordinary, + testDir: './import-tests', + testMatch: 'cv-import-docling.spec.ts', + timeout: 180_000, + retries: 0, + webServer: [{ ...servers[0], env: { ...servers[0].env, Workers__AiOperationsEnabled: 'true', Ai__BaseUrl: process.env.CV_IMPORT_SIDECAR_URL, Ai__ProbeEnabled: 'false', Email__Enabled: 'false' } }, ...servers.slice(1)], +}); diff --git a/job-tracker-ui/src/cv-import-review.test.tsx b/job-tracker-ui/src/cv-import-review.test.tsx new file mode 100644 index 0000000..32f5f0d --- /dev/null +++ b/job-tracker-ui/src/cv-import-review.test.tsx @@ -0,0 +1,139 @@ +import React from 'react'; +import '@testing-library/jest-dom'; +import { fireEvent, render, screen, waitFor, within } from '@testing-library/react'; +import { api } from './api'; +import { emptyStructuredCv } from './profileCv'; +import { ToastProvider } from './toast'; +import { I18nProvider } from './i18n/I18nProvider'; +import CareerProfilePage from './views/CareerProfilePage'; + +jest.mock('./api', () => ({ + getApiErrorMessage: jest.requireActual('./api').getApiErrorMessage, + api: { get: jest.fn(), post: jest.fn(), put: jest.fn() }, +})); +jest.mock('./components/CropImageDialog', () => () => null); + +const mockedApi = api as jest.Mocked; +const run = { id: 23, trigger: 'upload', status: 'pending_review', artifactFileName: 'career.pdf', + startedAtUtc: '2026-10-26T10:00:00Z', parserVersion: 'docling-v1', normalizerVersion: 'document-v1', llmPromptVersion: 'none' }; +const review = { runId: 23, status: 'pending_review', revision: 7, fileName: 'career.pdf', + sourceUrl: '/api/profile-cv/runs/23/source', sourceText: 'Source name\nDocker', warnings: ['Check ambiguous dates.'], + groups: [{ id: 'Contact', category: 'Contact', label: 'Contact', matchDescription: null, fields: [ + { id: 'Contact.fullName', label: 'Full name', currentValue: 'Saved name', extractedValue: 'Source name', sourceSnippet: 'Source name', sourcePage: 1, method: 'docling', reviewState: 'suggested' }, + { id: 'Contact.headline', label: 'Headline', currentValue: 'Saved headline', extractedValue: 'Engineer', sourceSnippet: null, sourcePage: null, method: 'deterministic', reviewState: 'suggested' }, + ] }] }; +let currentReview = review; +let runs = [run]; + +function renderImport() { + window.history.replaceState({}, '', '/career?section=import'); + return render(); +} + +beforeEach(() => { + window.localStorage.clear(); + currentReview = review; + runs = [run]; + URL.createObjectURL = jest.fn(() => 'blob:owned-source'); + URL.revokeObjectURL = jest.fn(); + mockedApi.get.mockImplementation((url: string) => { + if (url === '/career/profile') return Promise.resolve({ data: { profile: emptyStructuredCv(), revision: 7, completeness: { percent: 0, missing: [], sections: [] } } } as any); + if (url === '/auth/me') return Promise.resolve({ data: { provider: 'local', userName: 'Importer' } } as any); + if (url === '/career/profile/versions') return Promise.resolve({ data: [] } as any); + if (url === '/profile-cv/runs') return Promise.resolve({ data: runs } as any); + if (url === '/profile-cv/runs/23/review') return Promise.resolve({ data: currentReview } as any); + if (url === '/profile-cv/runs/23/diff') return Promise.resolve({ data: { revision: 7, diff: { categories: [], hasChanges: false, totalAdded: 0, totalUpdated: 0, totalLowConfidence: 0 } } } as any); + if (url === '/profile-cv/runs/23/source') return Promise.resolve({ data: new Blob(['%PDF-1.4'], { type: 'application/pdf' }) } as any); + return Promise.resolve({ data: {} } as any); + }); + mockedApi.post.mockResolvedValue({ data: { runId: 23, status: 'applied', revision: 8, structuredCv: emptyStructuredCv() } } as any); +}); + +afterEach(() => jest.clearAllMocks()); + +test('Docling review requires explicit decisions and sends only reviewed corrections before showing durable completion', async () => { + renderImport(); + const choice = await screen.findByRole('combobox', { name: 'Decision: Full name' }); + expect(choice).toHaveValue(''); + expect(screen.getByRole('button', { name: 'Apply reviewed changes' })).toBeDisabled(); + expect(screen.getByText('Current: Saved name')).toBeInTheDocument(); + expect(screen.getByText('Imported: Source name')).toBeInTheDocument(); + expect(screen.getByText('Page 1')).toBeInTheDocument(); + expect(screen.getByText('Check ambiguous dates.')).toBeInTheDocument(); + fireEvent.change(choice, { target: { value: 'correct' } }); + fireEvent.change(screen.getByLabelText('Correction: Full name'), { target: { value: 'Reviewed name' } }); + fireEvent.change(screen.getByRole('combobox', { name: 'Decision: Professional headline' }), { target: { value: 'keep' } }); + fireEvent.click(screen.getByRole('button', { name: 'Apply reviewed changes' })); + await waitFor(() => expect(mockedApi.post).toHaveBeenCalledWith('/profile-cv/runs/23/apply-reviewed', { + expectedVersion: 7, decisions: [{ id: 'Contact.fullName', action: 'correct', value: 'Reviewed name' }, { id: 'Contact.headline', action: 'keep' }], + })); + expect(await screen.findByText('Reviewed changes saved to your Career Profile.')).toBeInTheDocument(); + expect(screen.getByRole('link', { name: 'Open Career Profile' })).toHaveAttribute('href', '/career?section=profile'); + expect(screen.getByRole('link', { name: 'Open CV editor' })).toHaveAttribute('href', '/career/builder'); +}); + +test('stale apply retains corrections and refreshes current values but requires every choice to be confirmed again', async () => { + mockedApi.post.mockRejectedValueOnce({ response: { status: 409 } }); + renderImport(); + fireEvent.change(await screen.findByRole('combobox', { name: 'Decision: Full name' }), { target: { value: 'correct' } }); + fireEvent.change(screen.getByLabelText('Correction: Full name'), { target: { value: 'Keep this correction' } }); + fireEvent.change(screen.getByRole('combobox', { name: 'Decision: Professional headline' }), { target: { value: 'use' } }); + currentReview = { ...review, revision: 9, groups: [{ ...review.groups[0], fields: [{ ...review.groups[0].fields[0], currentValue: 'Changed elsewhere' }, review.groups[0].fields[1]] }] }; + fireEvent.click(screen.getByRole('button', { name: 'Apply reviewed changes' })); + expect(await screen.findByText('Current: Changed elsewhere')).toBeInTheDocument(); + expect(screen.getByLabelText('Correction: Full name')).toHaveValue('Keep this correction'); + expect(screen.getByRole('button', { name: 'Apply reviewed changes' })).toBeDisabled(); + expect(screen.getByRole('combobox', { name: 'Decision: Full name' })).toHaveValue(''); + fireEvent.change(screen.getByRole('combobox', { name: 'Decision: Full name' }), { target: { value: 'correct' } }); + expect(screen.getByRole('button', { name: 'Apply reviewed changes' })).toBeDisabled(); + fireEvent.change(screen.getByRole('combobox', { name: 'Decision: Professional headline' }), { target: { value: 'keep' } }); + fireEvent.click(screen.getByRole('button', { name: 'Apply reviewed changes' })); + await waitFor(() => expect(mockedApi.post).toHaveBeenLastCalledWith('/profile-cv/runs/23/apply-reviewed', { + expectedVersion: 9, decisions: [{ id: 'Contact.fullName', action: 'correct', value: 'Keep this correction' }, { id: 'Contact.headline', action: 'keep' }], + })); +}); + +test('original PDF uses the authenticated API blob and revokes its URL when leaving the page', async () => { + const { unmount } = renderImport(); + const preview = await screen.findByTitle('Original CV document'); + expect(preview).toHaveAttribute('src', 'blob:owned-source'); + expect(mockedApi.get).toHaveBeenCalledWith('/profile-cv/runs/23/source', expect.objectContaining({ responseType: 'blob' })); + expect(screen.getByRole('link', { name: 'Download original' })).toHaveAttribute('download', 'career.pdf'); + unmount(); + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:owned-source'); +}); + +test.each([['career.pdf', 'application/pdf', '/profile-cv/import'], ['career.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', '/profile-cv/import'], ['career.txt', 'text/plain', '/profile-cv/upload']])('upload %s uses its compatible pipeline', async (fileName, type, route) => { + runs = []; + const { container } = renderImport(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Upload CV' })).toBeEnabled()); + fireEvent.change(container.querySelector('input[type="file"][accept*=".pdf"]')!, { target: { files: [new File(['content'], fileName, { type })] } }); + await waitFor(() => expect(mockedApi.post).toHaveBeenCalledWith(route, expect.any(FormData), { headers: { 'Content-Type': 'multipart/form-data' } })); +}); + +test('oversized document is explained before upload and a later valid file remains selectable', async () => { + runs = []; + const { container } = renderImport(); + await screen.findByRole('button', { name: 'Upload CV' }); + const input = container.querySelector('input[type="file"][accept*=".pdf"]')!; + const file = new File(['small'], 'career.pdf', { type: 'application/pdf' }); + Object.defineProperty(file, 'size', { value: 5 * 1024 * 1024 + 1 }); + fireEvent.change(input, { target: { files: [file] } }); + expect(await screen.findByText('Choose a file no larger than 5 MiB.')).toBeInTheDocument(); + expect(mockedApi.post).not.toHaveBeenCalled(); + fireEvent.change(input, { target: { files: [new File(['pdf'], 'career.pdf', { type: 'application/pdf' })] } }); + await waitFor(() => expect(mockedApi.post).toHaveBeenCalledTimes(1)); +}); + +test('group approval is deliberate and a dirty profile disables application without losing review corrections', async () => { + renderImport(); + const group = await screen.findByRole('group', { name: 'Contact information' }); + fireEvent.click(within(group).getByRole('button', { name: 'Use all extracted values in this group' })); + expect(screen.getByRole('button', { name: 'Apply reviewed changes' })).toBeEnabled(); + fireEvent.click(screen.getByRole('tab', { name: 'Career profile' })); + fireEvent.change(await screen.findByLabelText('Full name'), { target: { value: 'Unsaved draft' } }); + fireEvent.click(screen.getByRole('tab', { name: 'Import review' })); + expect(screen.getByRole('button', { name: 'Apply reviewed changes' })).toBeDisabled(); + fireEvent.click(screen.getByRole('tab', { name: 'Career profile' })); + expect(screen.getByLabelText('Full name')).toHaveValue('Unsaved draft'); +}); diff --git a/job-tracker-ui/src/i18n/translations.ts b/job-tracker-ui/src/i18n/translations.ts index a060713..a7c0d08 100644 --- a/job-tracker-ui/src/i18n/translations.ts +++ b/job-tracker-ui/src/i18n/translations.ts @@ -2,6 +2,58 @@ export type UiLanguage = "en" | "nb"; export const translations = { en: { + cvImportReviewHelp: "Review every field. Nothing changes in your Career Profile until you apply your choices.", + cvImportPanels: "Import source and review", + cvImportSourcePanel: "Source document", + cvImportReviewPanel: "Review extracted information", + cvImportDownload: "Download original", + cvImportSourceTitle: "Original CV document", + cvImportSourceLoading: "Loading original document", + cvImportSourceFailed: "Could not load the original document. Source excerpts remain available for review.", + cvImportSourceEmpty: "No source text is available. Download the original document to check it.", + cvImportEvidence: "Source excerpt", + cvImportPage: "Page {page}", + cvImportNoEvidence: "No source excerpt available. Check the original before approving this value.", + cvImportMethodDocument: "Read from document", + cvImportMethodDeterministic: "Suggested from document structure", + cvImportReviewedCount: "{reviewed} of {total} fields reviewed", + cvImportDecision: "Decision: {field}", + cvImportChoose: "Choose a decision", + cvImportKeep: "Keep current value / skip addition", + cvImportUse: "Use extracted value", + cvImportCorrect: "Correct extracted value", + cvImportCorrection: "Correction: {field}", + cvImportUseGroup: "Use all extracted values in this group", + cvImportApplyReviewed: "Apply reviewed changes", + cvImportStaleReview: "Your Career Profile changed elsewhere. Your corrections are retained. Compare the refreshed current values and confirm every choice again.", + cvImportNoFields: "No supported fields were extracted. Your Career Profile has not changed.", + cvImportSaved: "Reviewed changes saved to your Career Profile.", + cvImportOpenProfile: "Open Career Profile", + cvImportOpenEditor: "Open CV editor", + cvImportSizeError: "Choose a file no larger than 5 MiB.", + cvImportFormatError: "Choose a PDF, DOCX, text, Markdown, PNG, JPEG or WebP file.", + cvImportUploadHelp: "Drop a PDF or DOCX here, or choose a file. Maximum 5 MiB. Text, Markdown and images use the existing importer.", + cvImportStageExtracting: "Reading document structure", + cvImportStageValidating: "Validating your document", + cvImportFieldName: "Name", + cvImportFieldRole: "Role", + cvImportMatchedRecord: "Matches an existing record — review the affected fields", + cvImportNewRecord: "New record — keep only the information you want to add", + cvImportExpired: "Source expired — upload the document again", + cvImportStageSections: "Identifying document sections", + cvImportStageAnalysing: "Analysing career information", + cvImportStageDetails: "Validating extracted details", + cvImportStagePreparing: "Preparing your review", + cvImportWarningNative: "Complex PDF columns and table relationships may need corrections. Check the original document.", + cvImportWarningPages: "DOCX page numbers cannot be established reliably. Source excerpts are provided instead.", + cvImportWarningOcr: "OCR is disabled. Scanned or image-only content cannot be extracted.", + cvImportStageInterpreting: "Preparing extracted information for review", + cvImportStageReady: "Ready for your review", + cvImportFieldTitle: "Title", + cvImportFieldCompany: "Company", + cvImportFieldStart: "Start date", + cvImportFieldEnd: "End date", + cvImportFieldCurrent: "Current position", appTitle: "Jobbjakt", appTagline: "Track your hunt", applicationLanguage: "Application language", @@ -2927,6 +2979,58 @@ export const translations = { coverAiReject: "Reject", }, nb: { + cvImportReviewHelp: "Gå gjennom hvert felt. Karriereprofilen endres først når du bruker valgene dine.", + cvImportPanels: "Kilde og gjennomgang av import", + cvImportSourcePanel: "Kildedokument", + cvImportReviewPanel: "Se gjennom hentet informasjon", + cvImportDownload: "Last ned originalen", + cvImportSourceTitle: "Opprinnelig CV-dokument", + cvImportSourceLoading: "Laster originaldokumentet", + cvImportSourceFailed: "Kunne ikke laste originaldokumentet. Kildeutdragene er fortsatt tilgjengelige for gjennomgang.", + cvImportSourceEmpty: "Ingen kildetekst er tilgjengelig. Last ned originaldokumentet for å kontrollere det.", + cvImportEvidence: "Kildeutdrag", + cvImportPage: "Side {page}", + cvImportNoEvidence: "Ingen kildeutdrag tilgjengelig. Kontroller originalen før du godkjenner verdien.", + cvImportMethodDocument: "Lest fra dokumentet", + cvImportMethodDeterministic: "Foreslått ut fra dokumentstrukturen", + cvImportReviewedCount: "{reviewed} av {total} felt gjennomgått", + cvImportDecision: "Valg: {field}", + cvImportChoose: "Velg en handling", + cvImportKeep: "Behold nåværende verdi / hopp over tillegg", + cvImportUse: "Bruk hentet verdi", + cvImportCorrect: "Korriger hentet verdi", + cvImportCorrection: "Korrigering: {field}", + cvImportUseGroup: "Bruk alle hentede verdier i denne gruppen", + cvImportApplyReviewed: "Bruk gjennomgåtte endringer", + cvImportStaleReview: "Karriereprofilen ble endret et annet sted. Korrigeringene dine er beholdt. Sammenlign de oppdaterte verdiene og bekreft hvert valg på nytt.", + cvImportNoFields: "Ingen støttede felt ble hentet. Karriereprofilen er ikke endret.", + cvImportSaved: "Gjennomgåtte endringer er lagret i karriereprofilen.", + cvImportOpenProfile: "Åpne karriereprofilen", + cvImportOpenEditor: "Åpne CV-editoren", + cvImportSizeError: "Velg en fil som ikke er større enn 5 MiB.", + cvImportFormatError: "Velg en PDF-, DOCX-, tekst-, Markdown-, PNG-, JPEG- eller WebP-fil.", + cvImportUploadHelp: "Slipp en PDF eller DOCX her, eller velg en fil. Maksimalt 5 MiB. Tekst, Markdown og bilder bruker den eksisterende importen.", + cvImportStageExtracting: "Leser dokumentstrukturen", + cvImportStageValidating: "Kontrollerer dokumentet ditt", + cvImportFieldName: "Navn", + cvImportFieldRole: "Rolle", + cvImportMatchedRecord: "Samsvarer med en eksisterende oppføring — gjennomgå feltene som berøres", + cvImportNewRecord: "Ny oppføring — behold bare informasjonen du vil legge til", + cvImportExpired: "Kilden er utløpt — last opp dokumentet på nytt", + cvImportStageSections: "Identifiserer dokumentdelene", + cvImportStageAnalysing: "Analyserer karriereinformasjon", + cvImportStageDetails: "Kontrollerer de hentede opplysningene", + cvImportStagePreparing: "Klargjør gjennomgangen din", + cvImportWarningNative: "Komplekse PDF-kolonner og tabeller kan kreve rettelser. Kontroller originaldokumentet.", + cvImportWarningPages: "Sidetall i DOCX kan ikke fastslås pålitelig. Kildeutdrag vises i stedet.", + cvImportWarningOcr: "OCR er deaktivert. Skannet innhold eller innhold som bare består av bilder, kan ikke hentes ut.", + cvImportStageInterpreting: "Klargjør hentet informasjon for gjennomgang", + cvImportStageReady: "Klar for gjennomgang", + cvImportFieldTitle: "Stilling", + cvImportFieldCompany: "Arbeidsgiver", + cvImportFieldStart: "Startdato", + cvImportFieldEnd: "Sluttdato", + cvImportFieldCurrent: "Nåværende stilling", appTitle: "Jobbjakt", appTagline: "Hold oversikt over jobbsøkingen", applicationLanguage: "Språk i appen", diff --git a/job-tracker-ui/src/profile-page.test.tsx b/job-tracker-ui/src/profile-page.test.tsx index bd62bf2..1dccb85 100644 --- a/job-tracker-ui/src/profile-page.test.tsx +++ b/job-tracker-ui/src/profile-page.test.tsx @@ -582,14 +582,14 @@ test('/career profile exposes extraction progress', async () => { expect(fileInput).not.toBeNull(); }); -test('/career profile can queue a CV upload without visiting import review first', async () => { +test('/career profile can queue a legacy text CV upload without visiting import review first', async () => { extractionRunsResponse = []; mockedApi.post.mockResolvedValueOnce({ data: { queued: true, extractionRunId: 22, status: 'queued', created: true } } as any); const { container } = renderPage('profile'); expect(await screen.findByRole('button', { name: /upload cv/i })).toBeEnabled(); const fileInput = container.querySelector('input[type="file"][accept*=".pdf"]')!; - fireEvent.change(fileInput, { target: { files: [new File(['resume'], 'resume.pdf', { type: 'application/pdf' })] } }); + fireEvent.change(fileInput, { target: { files: [new File(['resume'], 'resume.txt', { type: 'text/plain' })] } }); await waitFor(() => expect(mockedApi.post).toHaveBeenCalledWith('/profile-cv/upload', expect.any(FormData), { headers: { 'Content-Type': 'multipart/form-data' }, diff --git a/job-tracker-ui/src/profileCv.test.ts b/job-tracker-ui/src/profileCv.test.ts index cb775f1..0b17dce 100644 --- a/job-tracker-ui/src/profileCv.test.ts +++ b/job-tracker-ui/src/profileCv.test.ts @@ -28,6 +28,11 @@ test("legacy structured items without IDs remain usable and acquire no client-ge } }); +test("reviewed field provenance survives client normalization and a later save", () => { + const source = { metadata: { fields: { "skills.0": { sourceBlockId: "b0001", sourceArtifactId: 21, sourceExtractionRunId: 7, sourceFileSha256: "abc", method: "deterministic-source" } } }, skills: ["C#"] }; + expect(normalizeStructuredCv(JSON.parse(JSON.stringify(source))).metadata.fields["skills.0"]).toMatchObject(source.metadata.fields["skills.0"]); +}); + test("legacy parsed sections keep their facts without inventing structured item IDs", () => { const profile = parseStructuredCvJson(JSON.stringify([ { name: "Professional Summary", content: "Recorded delivery\nRecorded support", wordCount: 4 }, diff --git a/job-tracker-ui/src/profileCv.ts b/job-tracker-ui/src/profileCv.ts index 5e56a68..56202f4 100644 --- a/job-tracker-ui/src/profileCv.ts +++ b/job-tracker-ui/src/profileCv.ts @@ -10,6 +10,9 @@ export type StructuredCvFieldMetadata = { sourceSnippet?: string; sourcePage?: number; sourceBlockId?: string; + sourceArtifactId?: number; + sourceExtractionRunId?: number; + sourceFileSha256?: string; reviewState?: string; lastUpdatedAtUtc?: string; }; @@ -225,6 +228,9 @@ export function normalizeStructuredCv(value: unknown): StructuredCvProfile { sourceSnippet: normalizeString(value?.sourceSnippet), sourcePage: value?.sourcePage != null && Number.isFinite(Number(value.sourcePage)) ? Number(value.sourcePage) : undefined, sourceBlockId: normalizeString(value?.sourceBlockId), + sourceArtifactId: value?.sourceArtifactId != null && Number.isInteger(Number(value.sourceArtifactId)) && Number(value.sourceArtifactId) > 0 ? Number(value.sourceArtifactId) : undefined, + sourceExtractionRunId: value?.sourceExtractionRunId != null && Number.isInteger(Number(value.sourceExtractionRunId)) && Number(value.sourceExtractionRunId) > 0 ? Number(value.sourceExtractionRunId) : undefined, + sourceFileSha256: normalizeString(value?.sourceFileSha256), reviewState: normalizeString(value?.reviewState), lastUpdatedAtUtc: normalizeString(value?.lastUpdatedAtUtc), }])) diff --git a/job-tracker-ui/src/views/CareerProfilePage.tsx b/job-tracker-ui/src/views/CareerProfilePage.tsx index 40e5c2d..c2f00ed 100644 --- a/job-tracker-ui/src/views/CareerProfilePage.tsx +++ b/job-tracker-ui/src/views/CareerProfilePage.tsx @@ -12,6 +12,7 @@ import CropImageDialog from "../components/CropImageDialog"; import ProfileCompleteness from "./career/ProfileCompleteness"; import CareerWorkspaceOverview from "./career/CareerWorkspaceOverview"; import CareerEvidenceBank from "./career/CareerEvidenceBank"; +import CvImportReview from "./career/CvImportReview"; import { EducationSection, InterestsSection, @@ -106,7 +107,7 @@ const importLabelKeys: Record = { "waiting to retry": "careerImportWaitingRetry", "waiting for approved fallback": "careerImportWaitingFallback", "cancellation requested": "operationsCancellationRequested", pending_review: "careerImportPendingReview", applied: "careerImportApplied", discarded: "careerImportDiscarded", failed: "careerImportFailed", cancelled: "careerImportCancelled", - upload: "careerImportUpload", rebuild: "careerImportRebuild", improve: "careerImportImprove", reprocess: "careerImportReprocess", + upload: "careerImportUpload", document_import: "careerImportUpload", expired: "cvImportExpired", rebuild: "careerImportRebuild", improve: "careerImportImprove", reprocess: "careerImportReprocess", }; type MeResponse = { @@ -178,6 +179,7 @@ export default function CareerProfilePage() { const [profileRevision, setProfileRevision] = useState(null); const profileReadId = useRef(0); const [uploadingCv, setUploadingCv] = useState(false); + const [uploadError, setUploadError] = useState(null); const [improvingCv, setImprovingCv] = useState(false); const [rebuildingCv, setRebuildingCv] = useState(false); const [uploadingAvatar, setUploadingAvatar] = useState(false); @@ -323,19 +325,32 @@ export default function CareerProfilePage() { }, []); const queueCvUpload = useCallback(async (file: File) => { + if (uploadingCv || !canUseAi || me?.provider !== "local") return; + if (file.size > 5 * 1024 * 1024) { + setUploadError(t("cvImportSizeError")); + return; + } + if (!/\.(pdf|docx|txt|md|png|jpe?g|webp)$/i.test(file.name)) { + setUploadError(t("cvImportFormatError")); + return; + } const formData = new FormData(); formData.append("file", file); + setUploadError(null); setUploadingCv(true); try { - const res = await api.post("/profile-cv/upload", formData, { headers: { "Content-Type": "multipart/form-data" } }); + const route = /\.(pdf|docx)$/i.test(file.name) ? "/profile-cv/import" : "/profile-cv/upload"; + const res = await api.post(route, formData, { headers: { "Content-Type": "multipart/form-data" } }); await loadExtractionRuns(); toast(t("profileCvQueuedUpload", { id: res.data.extractionRunId }), "info"); } catch (error) { - toast(getApiErrorMessage(error, t("profileCvUploadFailed")), "error"); + const message = getApiErrorMessage(error, t("profileCvUploadFailed")); + setUploadError(message); + toast(message, "error"); } finally { setUploadingCv(false); } - }, [loadExtractionRuns, t, toast]); + }, [canUseAi, loadExtractionRuns, me?.provider, t, toast, uploadingCv]); const loadProfile = useCallback(async () => { const readId = ++profileReadId.current; @@ -389,7 +404,7 @@ export default function CareerProfilePage() { useEffect(() => { for (const run of extractionRuns) { - if (run.status === "pending_review" && !runDiffs[run.id] && !runDiffLoading[run.id] && runDiffErrors[run.id] === undefined) void loadRunDiff(run.id); + if (run.parserVersion !== "docling-v1" && run.status === "pending_review" && !runDiffs[run.id] && !runDiffLoading[run.id] && runDiffErrors[run.id] === undefined) void loadRunDiff(run.id); } }, [extractionRuns, loadRunDiff, runDiffErrors, runDiffLoading, runDiffs]); @@ -486,6 +501,8 @@ export default function CareerProfilePage() { : run.status === "queued" || run.status === "running"); const pendingCvRun = extractionRuns.find((run) => run.status === "pending_review"); const extractionProgress = activeCvRun?.operation?.progressPercent; + const stageKeys: Record = { cv_validating_file: "cvImportStageValidating", cv_extracting: "cvImportStageExtracting", cv_identifying_sections: "cvImportStageSections", cv_analysing: "cvImportStageAnalysing", cv_validating_details: "cvImportStageDetails", cv_preparing_review: "cvImportStagePreparing", cv_interpreting: "cvImportStageInterpreting", cv_review_ready: "cvImportStageReady" }; + const importStage = (stage?: string | null) => stage && Object.hasOwn(stageKeys, stage) ? t(stageKeys[stage]) : t("careerProfileImportProcessing"); const importLabel = (value: string) => Object.hasOwn(importLabelKeys, value) ? t(importLabelKeys[value]) : value; const importChangeLabel = (change: CvImportChange, category: string) => category === "Professional summary" ? importLabel(category) : change.id === "Contact" && change.confidence === "Low" && change.label === "Contact details" ? t("careerImportContactDetails") : change.label; @@ -620,7 +637,7 @@ export default function CareerProfilePage() { {typeof extractionProgress === "number" ? t("careerProfileImportPercent", { percent: extractionProgress }) - : t("careerProfileImportProcessing")} + : importStage(activeCvRun?.operation?.progressStage)} : null} {pendingCvRun ? navigateWorkspace("import")}>{t("careerProfileImportReview")}}> @@ -695,7 +712,12 @@ export default function CareerProfilePage() { - + event.preventDefault()} onDrop={(event) => { + event.preventDefault(); + if (editorDisabled || uploadingCv || activeCvRun || !canUseAi) return; + const file = event.dataTransfer.files[0]; + if (file) void queueCvUpload(file); + }}> {!canUseAi && ( @@ -772,6 +794,8 @@ export default function CareerProfilePage() { {uploadingCv ? : null} + {t("cvImportUploadHelp")} + {uploadError ? {uploadError} : null} {t("profileCvStructuredDefaultHint")} @@ -819,6 +843,7 @@ export default function CareerProfilePage() { {run.artifactFileName || t("profileCvNoStoredArtifact")} + {run.operation?.progressStage && activeOperationStatuses.has(run.operation.status) ? {importStage(run.operation.progressStage)} : null} {run.parserVersion} · {new Date(run.startedAtUtc).toLocaleString(language === "nb" ? "nb-NO" : "en")} @@ -837,7 +862,21 @@ export default function CareerProfilePage() { {t(runActionErrors[run.id].key)} {runActionErrors[run.id].message ? {runActionErrors[run.id].message} : null} : null} - {run.status === "pending_review" ? ( + {run.parserVersion === "docling-v1" && (run.status === "pending_review" || run.status === "applied") ? { + profileMutationPending.current = applying; + setReviewingRunId(applying ? run.id : null); + setReviewAction(applying ? "accept" : null); + }} + onApplied={async () => { + setExtractionRuns((current) => current.map((item) => item.id === run.id ? { ...item, status: "applied" } : item)); + invalidateImportComparisons(); + await Promise.all([loadProfile(), loadExtractionRuns(), loadVersions()]); + }} + onDiscard={() => void reviewRun(run.id, "discard")} + /> : run.status === "pending_review" ? ( {runDiffs[run.id] ? ( <> diff --git a/job-tracker-ui/src/views/career/CvImportReview.tsx b/job-tracker-ui/src/views/career/CvImportReview.tsx new file mode 100644 index 0000000..ccd3451 --- /dev/null +++ b/job-tracker-ui/src/views/career/CvImportReview.tsx @@ -0,0 +1,220 @@ +import React, { useCallback, useEffect, useRef, useState } from "react"; +import { Alert, Box, Button, FormControl, InputLabel, LinearProgress, NativeSelect, Paper, Tab, Tabs, TextField, Typography } from "@mui/material"; +import { api, getApiErrorMessage } from "../../api"; +import { useI18n } from "../../i18n/I18nProvider"; +import type { TranslationKey } from "../../i18n/translations"; + +type ReviewField = { + id: string; label: string; currentValue: string | null; extractedValue: string; + sourceSnippet: string | null; sourcePage: number | null; method: string; reviewState: string; +}; +type ReviewGroup = { id: string; category: string; label: string; matchDescription: string | null; fields: ReviewField[] }; +type ImportReview = { + runId: number; status: string; revision: number; fileName: string; sourceText: string; + warnings: string[]; groups: ReviewGroup[]; +}; +type Choice = { action: "keep" | "use" | "correct" | "add"; value?: string; confirmed: boolean }; + +const labelKeys: Record = { + contact: "careerProfileContactSection", fullName: "profileCvContactFullName", headline: "profileCvContactHeadline", + email: "profileCvContactEmail", phone: "profileCvContactPhone", location: "profileCvContactLocation", + website: "profileCvContactWebsite", linkedIn: "profileCvContactLinkedIn", summary: "profileCvStructuredSummary", + jobs: "cvSectionExperience", education: "cvSectionEducation", skills: "cvSectionSkills", languages: "cvSectionLanguages", + interests: "cvSectionInterests", projects: "cvSectionProjects", certifications: "cvSectionCertifications", + title: "cvImportFieldTitle", company: "cvImportFieldCompany", start: "cvImportFieldStart", end: "cvImportFieldEnd", + isCurrent: "cvImportFieldCurrent", bullets: "careerImportDetails", details: "careerImportDetails", + awards: "cvSectionAwards", publications: "cvSectionPublications", organisations: "cvSectionOrganisations", references: "cvSectionReferences", + qualification: "profileCvEducationQualification", institution: "profileCvEducationInstitution", level: "profileCvLanguageLevel", notes: "profileCvLanguageNotes", + name: "cvImportFieldName", issuer: "careerImportIssuer", date: "careerImportDates", role: "cvImportFieldRole", otherSections: "cvEditorCustom", + Contact: "careerProfileContactSection", "Full name": "profileCvContactFullName", Headline: "profileCvContactHeadline", + Email: "profileCvContactEmail", Phone: "profileCvContactPhone", Location: "profileCvContactLocation", + Website: "profileCvContactWebsite", LinkedIn: "profileCvContactLinkedIn", Summary: "profileCvStructuredSummary", + Experience: "cvSectionExperience", Education: "cvSectionEducation", Skills: "cvSectionSkills", + Languages: "cvSectionLanguages", Interests: "cvSectionInterests", Projects: "cvSectionProjects", + Certifications: "cvSectionCertifications", Level: "profileCvLanguageLevel", Details: "careerImportDetails", + Bullets: "careerImportDetails", Title: "cvImportFieldTitle", Company: "cvImportFieldCompany", + Start: "cvImportFieldStart", End: "cvImportFieldEnd", "Is current": "cvImportFieldCurrent", +}; + +export default function CvImportReview({ runId, disabled, onApplyingChange, onApplied, onDiscard }: { + runId: number; disabled: boolean; onApplyingChange: (applying: boolean) => void; + onApplied: () => Promise; onDiscard: () => void; +}) { + const { t } = useI18n(); + const translate = useRef(t); + translate.current = t; + const [review, setReview] = useState(null); + const [choices, setChoices] = useState>({}); + const [loading, setLoading] = useState(true); + const [comparisonCurrent, setComparisonCurrent] = useState(false); + const [error, setError] = useState(null); + const [stale, setStale] = useState(false); + const [applying, setApplying] = useState(false); + const applyPending = useRef(false); + const [applied, setApplied] = useState(false); + const [mobilePanel, setMobilePanel] = useState("review"); + const [sourceUrl, setSourceUrl] = useState(null); + const [sourceIsPdf, setSourceIsPdf] = useState(false); + const [sourceError, setSourceError] = useState(false); + const [sourceAttempt, setSourceAttempt] = useState(0); + const generation = useRef(0); + const reviewStatus = review?.status; + + const loadReview = useCallback(async () => { + const requestGeneration = ++generation.current; + setLoading(true); + setComparisonCurrent(false); + setError(null); + try { + const response = await api.get(`/profile-cv/runs/${runId}/review`); + if (requestGeneration !== generation.current) return; + setReview(response.data); + setApplied(response.data.status === "applied"); + setComparisonCurrent(true); + } catch (failure) { + if (requestGeneration === generation.current) setError(getApiErrorMessage(failure, translate.current("careerImportComparisonFailed"))); + } finally { + if (requestGeneration === generation.current) setLoading(false); + } + }, [runId]); + + useEffect(() => { + const requestState = generation; + setChoices({}); + setStale(false); + void loadReview(); + return () => { ++requestState.current; }; + }, [loadReview]); + + useEffect(() => { + if (reviewStatus !== "pending_review" || applied) return; + const controller = new AbortController(); + let disposed = false; + let ownedUrl: string | null = null; + setSourceUrl(null); + setSourceError(false); + void api.get(`/profile-cv/runs/${runId}/source`, { responseType: "blob", signal: controller.signal }).then((response) => { + if (disposed) return; + ownedUrl = URL.createObjectURL(response.data); + setSourceIsPdf(response.data.type === "application/pdf"); + setSourceUrl(ownedUrl); + }).catch(() => { if (!disposed) setSourceError(true); }); + return () => { + disposed = true; + controller.abort(); + if (ownedUrl) URL.revokeObjectURL(ownedUrl); + }; + }, [runId, sourceAttempt, reviewStatus, applied]); + + const label = (value: string) => Object.hasOwn(labelKeys, value) ? t(labelKeys[value]) : value; + const warningKeys: Record = { pdf_native_order_no_layout_or_tables: "cvImportWarningNative", docx_page_numbers_unavailable: "cvImportWarningPages", ocr_disabled: "cvImportWarningOcr" }; + const fields = review?.groups.flatMap((group) => group.fields) ?? []; + const reviewedCount = fields.filter((field) => choices[field.id]?.confirmed).length; + const allReviewed = fields.length > 0 && reviewedCount === fields.length; + + async function apply() { + if (disabled || applyPending.current || !comparisonCurrent || !review || !allReviewed) return; + applyPending.current = true; + setApplying(true); + onApplyingChange(true); + setError(null); + try { + await api.post(`/profile-cv/runs/${runId}/apply-reviewed`, { + expectedVersion: review.revision, + decisions: fields.map((field) => { + const choice = choices[field.id]; + return { id: field.id, action: choice.action, ...(choice.action === "correct" ? { value: choice.value ?? field.extractedValue } : {}) }; + }), + }); + setApplied(true); + await onApplied(); + } catch (failure) { + if ((failure as { response?: { status?: number } }).response?.status === 409) { + setStale(true); + setChoices((current) => Object.fromEntries(Object.entries(current).map(([id, choice]) => [id, { ...choice, confirmed: false }]))); + await loadReview(); + } else setError(getApiErrorMessage(failure, translate.current("profileCvApplyChangesFailed"))); + } finally { + applyPending.current = false; + setApplying(false); + onApplyingChange(false); + } + } + + if (applied) return + {t("cvImportSaved")} + + + + + ; + + return + {loading ? : null} + {error ? void loadReview()} disabled={loading}>{t("retry")} : undefined}>{error} : null} + {stale ? {t("cvImportStaleReview")} : null} + {!review ? null : <> + {t("cvImportReviewHelp")} + {review.warnings.map((warning, index) => {Object.hasOwn(warningKeys, warning) ? t(warningKeys[warning]) : warning})} + setMobilePanel(value)} aria-label={t("cvImportPanels")} sx={{ display: { xs: "flex", md: "none" }, mb: 1 }}> + + + + + + {t("cvImportSourcePanel")} + {review.fileName} + {sourceUrl ? : null} + {sourceError ? setSourceAttempt((current) => current + 1)}>{t("retry")}}>{t("cvImportSourceFailed")} + : !sourceUrl ? : null} + {sourceUrl && sourceIsPdf ? + : {review.sourceText || t("cvImportSourceEmpty")}} + + + {t("cvImportReviewPanel")} + {t("cvImportReviewedCount", { reviewed: reviewedCount, total: fields.length })} + {review.groups.map((group) => + {label(group.label)} + {group.matchDescription ? {group.matchDescription === "Matched existing record" ? t("cvImportMatchedRecord") : group.matchDescription === "New record" ? t("cvImportNewRecord") : group.matchDescription} : null} + + {group.fields.map((field) => { + const choice = choices[field.id]; + const fieldKey = field.id.split(".").filter((part) => !/^\d+$/.test(part)).at(-1) ?? field.label; + const fieldLabel = label(Object.hasOwn(labelKeys, fieldKey) ? fieldKey : field.label); + const inputId = `cv-import-${runId}-${field.id}`; + return + {fieldLabel} + {t("careerImportCurrentValue", { value: field.currentValue || t("careerImportEmptyValue") })} + {t("careerImportIncomingValue", { value: field.extractedValue || t("careerImportEmptyValue") })} + + {t("cvImportEvidence")}{field.sourcePage ? ` · ${t("cvImportPage", { page: field.sourcePage })}` : ""} + {field.sourcePage ? {t("cvImportPage", { page: field.sourcePage })} : null} + {field.sourceSnippet || t("cvImportNoEvidence")} + {t(field.method === "docling" ? "cvImportMethodDocument" : "cvImportMethodDeterministic")} + + + {t("cvImportDecision", { field: fieldLabel })} + { + const action = event.target.value as Choice["action"]; + setChoices((current) => ({ ...current, [field.id]: { action, value: current[field.id]?.value ?? field.extractedValue, confirmed: Boolean(action) } })); + }}> + + + + + + + {choice?.action === "correct" ? setChoices((current) => ({ ...current, [field.id]: { ...current[field.id], value: event.target.value } }))} sx={{ mt: 2 }} /> : null} + ; + })} + )} + {fields.length === 0 ? {t("cvImportNoFields")} : null} + + + + + + + } + ; +} diff --git a/tools/summarizer/cv_import_ai.py b/tools/summarizer/cv_import_ai.py index a2d0abd..7c6d5e8 100644 --- a/tools/summarizer/cv_import_ai.py +++ b/tools/summarizer/cv_import_ai.py @@ -37,6 +37,21 @@ otherSections.N.title or otherSections.N.items.N. Relationships across ambiguous FIELD = re.compile(r"^(?:contact\.(?:fullName|headline|email|phone|location|website|linkedIn|gitHub)|contact\.links\.\d{1,3}\.(?:label|url)|(?:summary|skills|interests|awards|publications|organisations|references)\.\d{1,3}|(?:jobs|education|certifications|projects|languages)\.\d{1,3}\.(?:title|company|name|institution|qualification|qualificationLevel|issuer|role|location|start|end|date|level|notes|(?:bullets|skills|details)\.\d{1,3})|otherSections\.\d{1,3}\.(?:title|items\.\d{1,3}))$") +def _supported_field(field): + parts = field.split(".") + fields = { + "jobs": ({"title", "company", "location", "start", "end"}, {"bullets", "skills"}), + "education": ({"institution", "qualification", "qualificationLevel", "location", "start", "end"}, {"details"}), + "certifications": ({"name", "issuer", "location", "date"}, {"details"}), + "projects": ({"name", "role", "location", "start", "end"}, {"bullets", "skills"}), + "languages": ({"name", "level", "notes"}, set()), + } + if parts[0] not in fields: + return bool(FIELD.fullmatch(field)) + scalar, lists = fields[parts[0]] + return bool(FIELD.fullmatch(field)) and ((len(parts) == 3 and parts[2] in scalar) or (len(parts) == 4 and parts[2] in lists)) + + def _validate_config(config): url = urlsplit(config.base_url) if url.scheme != "http" or url.username or url.password or url.query or url.fragment or url.path not in ("", "/") or not config.model or len(config.model) > 200: @@ -88,7 +103,8 @@ def analyse(blocks, config=None, generate=None): for item in parsed["suggestions"]: if not isinstance(item, dict) or set(item) != {"field", "value", "block_id", "quote"} or any(not isinstance(value, str) or not value.strip() or len(value) > 8000 for value in item.values()): raise ValueError("Invalid model suggestion.") - if not FIELD.fullmatch(item["field"]) or item["field"] in seen or item["block_id"] not in by_id or clean(item["quote"]) not in clean(by_id[item["block_id"]]) or clean(item["value"]) not in clean(item["quote"]): + grounded_value = re.search(r"(? None: if node.tag.rsplit("}", 1)[-1] in {"altChunk", "OLEObject", "object"}: raise ParserError("external_content") if node.attrib.get("TargetMode", "").lower() == "external": + # Hyperlinks are inert source data; converters cannot follow them in the sandbox. + from urllib.parse import urlsplit + if node.attrib.get("Type", "").endswith("/hyperlink") and urlsplit(node.attrib.get("Target", "")).scheme.lower() in {"http", "https", "mailto"}: + continue raise ParserError("external_content") target = node.attrib.get("Target", "") if ":" in target or "\\" in target or target.startswith("/"): @@ -265,7 +269,7 @@ def _validate_import_pdf(data: bytes) -> int: count = len(reader.pages) if count > MAX_PDF_PAGES: raise ParserError("pdf_page_limit") - forbidden = {"/OpenAction", "/AA", "/JS", "/JavaScript", "/Launch", "/GoToR", "/URI", "/SubmitForm", "/ImportData", "/EmbeddedFiles", "/Filespec", "/RichMedia", "/XFA"} + forbidden = {"/OpenAction", "/AA", "/JS", "/JavaScript", "/Launch", "/GoToR", "/SubmitForm", "/ImportData", "/EmbeddedFiles", "/Filespec", "/RichMedia", "/XFA"} pending = [(reader.trailer, 0)] seen: set[int] = set() while pending: @@ -278,6 +282,9 @@ def _validate_import_pdf(data: bytes) -> int: if depth > 40 or len(seen) > 10_000: raise ParserError("pdf_object_limit") if isinstance(value, DictionaryObject): + from urllib.parse import urlsplit + if "/URI" in value and urlsplit(str(value["/URI"])).scheme.lower() not in {"http", "https", "mailto"}: + raise ParserError("external_content") if forbidden.intersection(value) or str(value.get("/S", "")) in forbidden or str(value.get("/Type", "")) in forbidden: raise ParserError("external_content") pending.extend((item, depth + 1) for item in value.values()) diff --git a/tools/summarizer/tests/test_cv_import_ai.py b/tools/summarizer/tests/test_cv_import_ai.py index 1f133d6..7b5bb44 100644 --- a/tools/summarizer/tests/test_cv_import_ai.py +++ b/tools/summarizer/tests/test_cv_import_ai.py @@ -27,6 +27,8 @@ def test_grounded_suggestion_is_explicitly_unverified_model_output(): {"field": "jobs.0.start", "value": "2020-01", "block_id": "b0001", "quote": "2020"}, {"field": "education.0.qualification", "value": "PhD", "block_id": "b0001", "quote": "Python"}, {"field": "metadata.owner", "value": "Python", "block_id": "b0001", "quote": "Python"}, + {"field": "jobs.0.issuer", "value": "Python", "block_id": "b0001", "quote": "Python"}, + {"field": "skills.0", "value": "Py", "block_id": "b0001", "quote": "Python"}, {"field": "skills.0", "value": "Python", "block_id": "missing", "quote": "Python"}, {"field": "skills.0", "value": "Python", "block_id": "b0001", "quote": "Python", "confidence": 100}, ]) diff --git a/tools/summarizer/tests/test_document_extractor.py b/tools/summarizer/tests/test_document_extractor.py index 47d378f..76cee79 100644 --- a/tools/summarizer/tests/test_document_extractor.py +++ b/tools/summarizer/tests/test_document_extractor.py @@ -11,6 +11,7 @@ from pathlib import Path import pytest from docx import Document from pypdf import PdfWriter +from pypdf.annotations import Link from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject, TextStringObject sys.path.insert(0, str(Path(__file__).resolve().parents[1])) @@ -103,6 +104,30 @@ def test_docx_rejects_external_relationships(): assert failure.value.code == "external_content" +def test_docx_allows_inert_hyperlinks_without_network_access(): + payload = io.BytesIO() + with zipfile.ZipFile(io.BytesIO(docx_bytes())) as source, zipfile.ZipFile(payload, "w", zipfile.ZIP_DEFLATED) as output: + for entry in source.infolist(): + body = source.read(entry) + if entry.filename == "word/_rels/document.xml.rels": + body = body.replace(b"", b'') + output.writestr(entry.filename, body) + assert parser_child.extract_document(".docx", payload.getvalue())["method"] == "docling-docx-v1" + + +def test_pdf_allows_inert_web_links_but_rejects_script_links(): + for target, allowed in [("https://portfolio.example.test", True), ("javascript:alert(1)", False)]: + writer = PdfWriter(clone_from=io.BytesIO(pdf_bytes())) + writer.add_annotation(page_number=0, annotation=Link(rect=(40, 700, 200, 720), url=target)) + output = io.BytesIO() + writer.write(output) + if allowed: + assert parser_child.extract_document(".pdf", output.getvalue())["method"] == "docling-native-pdf-v1" + else: + with pytest.raises(parser_child.ParserError): + parser_child.extract_document(".pdf", output.getvalue()) + + def test_docx_rejects_archive_traversal(): payload = io.BytesIO() with zipfile.ZipFile(io.BytesIO(docx_bytes())) as source, zipfile.ZipFile(payload, "w", zipfile.ZIP_DEFLATED) as output: