diff --git a/JobTrackerApi.Tests/CvProcessingOperationTests.cs b/JobTrackerApi.Tests/CvProcessingOperationTests.cs index 56ce432..bf2690f 100644 --- a/JobTrackerApi.Tests/CvProcessingOperationTests.cs +++ b/JobTrackerApi.Tests/CvProcessingOperationTests.cs @@ -105,6 +105,71 @@ public sealed class CvProcessingOperationTests Assert.Null(run.CompletedAtUtc); } + [Fact] + public async Task Cancellation_before_claim_and_retry_keep_extraction_history_in_sync() + { + await using var fixture = await Fixture.CreateAsync(); + await fixture.SeedUserAsync(); + + Guid operationId; + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + using var owner = scope.ServiceProvider.GetRequiredService().UseBackgroundUser("user-1"); + Assert.IsType(await CreateController(scope.ServiceProvider).Upload(File())); + operationId = (await scope.ServiceProvider.GetRequiredService().UserOperations.SingleAsync()).Id; + } + + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + using var owner = scope.ServiceProvider.GetRequiredService().UseBackgroundUser("user-1"); + var store = scope.ServiceProvider.GetRequiredService(); + Assert.True(await store.RequestCancellationAsync(operationId, default)); + + var cancelledRun = await scope.ServiceProvider.GetRequiredService().CvExtractionRuns.AsNoTracking().SingleAsync(); + Assert.Equal("cancelled", cancelledRun.Status); + Assert.NotNull(cancelledRun.CompletedAtUtc); + } + + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + using var owner = scope.ServiceProvider.GetRequiredService().UseBackgroundUser("user-1"); + Assert.True(await scope.ServiceProvider.GetRequiredService().RetryAsync(operationId, default)); + var queuedRun = await scope.ServiceProvider.GetRequiredService().CvExtractionRuns.AsNoTracking().SingleAsync(); + Assert.Equal("queued", queuedRun.Status); + Assert.Null(queuedRun.ErrorMessage); + Assert.Null(queuedRun.CompletedAtUtc); + } + + Assert.True(await fixture.Provider.GetRequiredService().RunOnceAsync(default)); + await using var verification = fixture.Provider.CreateAsyncScope(); + Assert.Equal("pending_review", (await verification.ServiceProvider.GetRequiredService() + .CvExtractionRuns.IgnoreQueryFilters().SingleAsync()).Status); + } + + [Fact] + public async Task Deadline_before_claim_fails_the_dormant_extraction_run() + { + await using var fixture = await Fixture.CreateAsync(); + await fixture.SeedUserAsync(); + + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + using var owner = scope.ServiceProvider.GetRequiredService().UseBackgroundUser("user-1"); + Assert.IsType(await CreateController(scope.ServiceProvider).Upload(File())); + await scope.ServiceProvider.GetRequiredService().UserOperations + .ExecuteUpdateAsync(setters => setters.SetProperty(item => item.DeadlineAtUtc, DateTime.UtcNow.AddMinutes(-1))); + } + + Assert.False(await fixture.Provider.GetRequiredService().RunOnceAsync(default)); + await using var verification = fixture.Provider.CreateAsyncScope(); + var db = verification.ServiceProvider.GetRequiredService(); + Assert.Equal(OperationStatuses.Failed, (await db.UserOperations.IgnoreQueryFilters().SingleAsync()).Status); + var run = await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync(); + Assert.Equal("failed", run.Status); + Assert.Contains("deadline", run.ErrorMessage, StringComparison.OrdinalIgnoreCase); + Assert.NotNull(run.CompletedAtUtc); + } + private static ProfileCvController CreateController(IServiceProvider services) { var controller = services.GetRequiredService(); diff --git a/JobTrackerApi/Services/UserOperationStore.cs b/JobTrackerApi/Services/UserOperationStore.cs index dc7e5d7..98f24f9 100644 --- a/JobTrackerApi/Services/UserOperationStore.cs +++ b/JobTrackerApi/Services/UserOperationStore.cs @@ -249,6 +249,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr cancellationToken); if (affected == 1 && !canRetry) { + await SynchronizeCvRunAsync(operation, OperationStatuses.Failed, message, now, cancellationToken); db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Failed, now)); await db.SaveChangesAsync(cancellationToken); } @@ -272,6 +273,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr cancellationToken); if (cancelled == 1) { + await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken); db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now)); await db.SaveChangesAsync(cancellationToken); if (transaction is not null) await transaction.CommitAsync(cancellationToken); @@ -303,6 +305,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr cancellationToken); if (affected == 1) { + await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken); db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now)); await db.SaveChangesAsync(cancellationToken); if (transaction is not null) await transaction.CommitAsync(cancellationToken); @@ -332,6 +335,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr var existingNotification = await db.UserNotifications.FirstOrDefaultAsync(item => item.OperationId == operationId, cancellationToken); if (existingNotification is not null) db.UserNotifications.Remove(existingNotification); await db.SaveChangesAsync(cancellationToken); + await SynchronizeCvRunAsync(operation, OperationStatuses.Queued, null, UtcNow, cancellationToken); if (transaction is not null) await transaction.CommitAsync(cancellationToken); return true; } @@ -393,6 +397,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr .SetProperty(item => item.LeaseExpiresAtUtc, (DateTime?)null), cancellationToken); if (affected == 1) { + await SynchronizeCvRunAsync(operation, status, message, now, cancellationToken); db.UserNotifications.Add(CreateTerminalNotification(operation, status, now)); await db.SaveChangesAsync(cancellationToken); if (transaction is not null) await transaction.CommitAsync(cancellationToken); @@ -405,6 +410,43 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr return await db.Database.BeginTransactionAsync(cancellationToken); } + private Task SynchronizeCvRunAsync( + UserOperation operation, + string operationStatus, + string? message, + DateTime now, + CancellationToken cancellationToken) + { + if (!string.Equals(operation.TaskType, CvProcessingQueue.TaskType, StringComparison.Ordinal) + || !string.Equals(operation.SubjectType, CvProcessingQueue.SubjectType, StringComparison.Ordinal) + || !int.TryParse(operation.SubjectId, System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, out var runId) + || runId <= 0) + { + return Task.FromResult(0); + } + + var runStatus = operationStatus switch + { + OperationStatuses.Cancelled => "cancelled", + OperationStatuses.Failed => "failed", + OperationStatuses.Queued => "queued", + _ => null, + }; + if (runStatus is null) return Task.FromResult(0); + var completedAt = OperationStatuses.IsTerminal(operationStatus) + ? new DateTimeOffset(DateTime.SpecifyKind(now, DateTimeKind.Utc)) + : (DateTimeOffset?)null; + + return db.CvExtractionRuns.IgnoreQueryFilters() + .Where(run => run.Id == runId && run.OwnerUserId == operation.OwnerUserId) + .ExecuteUpdateAsync(setters => setters + .SetProperty(run => run.Status, runStatus) + .SetProperty(run => run.ErrorMessage, message) + .SetProperty(run => run.CompletedAtUtc, completedAt), + cancellationToken); + } + private static UserNotification CreateTerminalNotification(UserOperation operation, string status, DateTime now) { var (kind, title, message) = status switch diff --git a/docs/audits/verification-log.md b/docs/audits/verification-log.md index db6bd4c..fa053cd 100644 --- a/docs/audits/verification-log.md +++ b/docs/audits/verification-log.md @@ -211,3 +211,4 @@ Output was reduced to filenames and commit counts. The token artifact appears un | V-177 | Python unit tests and plan-only execution against the checked-in synthetic fixture; script/content/SSRF/report review | Repository root | Prepare reproducible privacy-safe PROD-003 evaluation without authorizing a model or network call | PASS — harness tests 4/4; plan validates four Strategy cases across 4K/8K for eight future requests. Default performs no HTTP; exact model, `--execute` and explicit output are required; model pull/delete is absent; report excludes raw fixture/prompt/output content | No Ollama request, provider/internet access, model metadata query through the harness, candidate pull, inference or production change. Candidate metadata/licenses/results and model decision remain unmeasured | PROD-003 repository harness complete; execution remains blocked | | V-178 | Benchmark harness safety tests, plan-only execution and CI workflow inspection | Repository root | Prevent an approved private Ollama origin from escaping through proxy settings or redirects and make the boundary a release gate | PASS — 5/5 standard-library tests; proxy discovery is disabled, redirects are refused, plan-only output remains eight future Strategy requests, and CI now runs the suite without dependencies or network execution | No network, Ollama, provider, package or production call occurred | Benchmark request boundary corrected and CI-enforced | | V-179 | Account-deletion real-SQLite failure/retry tests; sidecar token/cache tests; full backend; Compose validation | Repository root / `tools/summarizer` | Remove the live sidecar-cache and shared tombstone-path gaps without enabling deletion | PASS — lifecycle 6/6, backend 658/658, sidecar 23/23 and Compose config pass. Sidecar failure withholds completion/tombstone until retry; maintenance purge is token-protected; tombstones map to a separate named volume; activation defaults false | Synthetic rows/cache only; no production volume, deletion, restart, provider revocation, backup restore or retention decision | SEC-009 repository cache/storage boundary complete; production activation remains blocked | +| V-180 | CV operation/store focused real-SQLite tests and full backend | Repository root | Keep dormant CV extraction history consistent with cancellation, deadline recovery and retry before worker claim | PASS — focused 17/17 and backend 660/660. Cancel sets the run terminal immediately, retry reopens it, deadline recovery fails it, and owner/task/subject predicates prevent unrelated updates | Synthetic rows only; no parser/model/MariaDB/production process interruption | AI-004 dormant-row consistency gap closed | diff --git a/docs/verification/ai-004-cv-processing-queue.md b/docs/verification/ai-004-cv-processing-queue.md index 1e298c1..aa82882 100644 --- a/docs/verification/ai-004-cv-processing-queue.md +++ b/docs/verification/ai-004-cv-processing-queue.md @@ -34,12 +34,14 @@ No dependency, schema, migration, proxy timeout or production switch changed. `W ## Remaining gates and known limits - SEC-006/007 still own fixed parser versions, page/pixel/decompression/memory/process isolation and complete parser-child cancellation/cleanup. Legacy structured parsing calls are not all cancellation-aware. No malicious file was executed. -- Cancellation of an operation before a worker claims it is authoritative in `UserOperation` and shown correctly after refresh; the underlying extraction row is reconciled when retried/processed, but immediate terminal synchronization of that dormant row remains follow-up cleanup work. +- Operation cancellation, terminal failure/deadline recovery, and explicit retry now synchronize the referenced owner-scoped extraction row immediately. A queued CV operation can no longer leave history stuck at `queued` after it is cancelled or expires before worker claim. - Browser localhost is denied by administrator policy. No real browser/mobile/theme/keyboard/refresh/back-forward workflow or screenshot is claimed. - The authorized private CV was not used. Synthetic input must pass the SEC-006/007 gates before that local-only check. - The generic lease tests cover restart recovery, but no CV parser/model process was interrupted and resumed in a runtime canary. - MariaDB, selected Ollama model, worker telemetry, production activation and rollback canary remain unverified. The worker stays default-off. +The focused CV/operation/store regression slice is now 17/17 and the full backend is 660/660 after dormant-row cancellation/deadline/retry coverage. + ## Rollback Keep `Workers:AiOperationsEnabled=false`, revert `c3c5af8`, and retain the additive operation/extraction tables. Cancel or drain queued `cv.process` operations before removing the handler. No database downgrade or artifact rewrite is required; existing extraction runs remain readable. diff --git a/docs/work-programmes/decisions.md b/docs/work-programmes/decisions.md index 4508a36..740b83e 100644 --- a/docs/work-programmes/decisions.md +++ b/docs/work-programmes/decisions.md @@ -779,3 +779,13 @@ - **Consequences:** a rare account deletion globally clears the shared summary cache because entries are not owner-keyed. An unavailable sidecar delays completion but does not restore deleted live rows/files. Production still needs protected volume custody, provider semantics, retention decisions and a disposable restore rehearsal. - **User approval required:** Production activation and operational mutation only. Repository implementation remains inert behind `ACCOUNT_DELETION_ENABLED=false`. - **Reversible:** Keep deletion disabled, drain any in-flight request, then revert the endpoint/client and Compose volume mapping. Never remove a deployed tombstone ledger while older restorable backups exist. + +## DEC-079 — Synchronize durable CV terminal state centrally + +- **Date:** 2026-08-15 +- **Decision:** When the shared operation store cancels, terminally fails, recovers an expired deadline, or retries a `cv.process` operation, update its explicitly referenced owner-scoped `CvExtractionRun` in the same transaction boundary. +- **Reason/evidence:** cancellation before worker claim was authoritative in `UserOperation` but left the domain history row queued indefinitely. Focused 17/17 tests cover cancel-before-claim, retry, deadline recovery and existing lease behavior; backend 660/660 passes. +- **Alternatives considered:** reconcile only in the controller; wait for a later worker; add a generic observer framework. These miss recovery paths, preserve stale history, or overbuild a two-task operation system. +- **Consequences:** generic operations remain independent; the one existing persisted subject projection is synchronized through a narrow task/subject check with an explicit owner predicate. +- **User approval required:** No; local consistency fix with no schema, dependency or production change. +- **Reversible:** Revert the store helper/tests. No stored format changed. diff --git a/docs/work-programmes/master-work-plan.md b/docs/work-programmes/master-work-plan.md index e421536..471caa2 100644 --- a/docs/work-programmes/master-work-plan.md +++ b/docs/work-programmes/master-work-plan.md @@ -558,9 +558,9 @@ This queue records the highest-value work that can proceed without production cr - **Required production verification:** synthetic/local-only canary, no external payload, restart recovery. - **Status:** `IMPLEMENTED — NOT VERIFIED`. - **Blocker:** SEC-006 dependency upgrades need internet permission; browser/private-file/MariaDB/production reproduction remains unavailable. Synthetic repository work can continue. -- **Evidence:** `docs/verification/ai-004-cv-processing-queue.md`; V-104–V-107; real SQLite synthetic integration proves 202/active deduplication/owner-scoped handler/retry provenance/notification/review gate; backend 594/594; frontend 161/161 and build. +- **Evidence:** `docs/verification/ai-004-cv-processing-queue.md`; V-104–V-107/V-180; real SQLite synthetic integration proves 202/active deduplication/owner-scoped handler/retry provenance/notification/review gate and pre-claim cancellation/deadline synchronization; focused operation lifecycle 17/17; backend 660/660; frontend 161/161 and build. - **Commit:** `c3c5af8` (`feat(cv)!: queue durable processing`). -- **Remaining work:** SEC-006/007 parser dependency/isolation and complete parser cancellation; browser synthetic upload/refresh/retry/cancel/review at required widths/themes/keyboard; selected-model and worker-restart canary; MariaDB/production rollout. Reconcile dormant extraction-row status immediately when an operation is cancelled before claim. Do not use the private CV before safeguards. +- **Remaining work:** SEC-006/007 parser dependency/isolation and complete parser-child cancellation; browser synthetic upload/refresh/retry/cancel/review at required widths/themes/keyboard; selected-model and worker-restart canary; MariaDB/production rollout. Do not use the private CV before safeguards. ### UX-001 — Unified authentication page