fix(cv): sync operation terminal state
CI and Deploy / test (pull_request) Successful in 5m12s
CI and Deploy / deploy (pull_request) Has been skipped

This commit is contained in:
cesnimda
2026-08-15 19:45:44 +02:00
parent 7185491a05
commit dbff0f8d49
6 changed files with 123 additions and 3 deletions
@@ -105,6 +105,71 @@ public sealed class CvProcessingOperationTests
Assert.Null(run.CompletedAtUtc);
}
[Fact]
public async Task Cancellation_before_claim_and_retry_keep_extraction_history_in_sync()
{
await using var fixture = await Fixture.CreateAsync();
await fixture.SeedUserAsync();
Guid operationId;
await using (var scope = fixture.Provider.CreateAsyncScope())
{
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Upload(File()));
operationId = (await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().UserOperations.SingleAsync()).Id;
}
await using (var scope = fixture.Provider.CreateAsyncScope())
{
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
var store = scope.ServiceProvider.GetRequiredService<UserOperationStore>();
Assert.True(await store.RequestCancellationAsync(operationId, default));
var cancelledRun = await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().CvExtractionRuns.AsNoTracking().SingleAsync();
Assert.Equal("cancelled", cancelledRun.Status);
Assert.NotNull(cancelledRun.CompletedAtUtc);
}
await using (var scope = fixture.Provider.CreateAsyncScope())
{
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
Assert.True(await scope.ServiceProvider.GetRequiredService<UserOperationStore>().RetryAsync(operationId, default));
var queuedRun = await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().CvExtractionRuns.AsNoTracking().SingleAsync();
Assert.Equal("queued", queuedRun.Status);
Assert.Null(queuedRun.ErrorMessage);
Assert.Null(queuedRun.CompletedAtUtc);
}
Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
await using var verification = fixture.Provider.CreateAsyncScope();
Assert.Equal("pending_review", (await verification.ServiceProvider.GetRequiredService<JobTrackerContext>()
.CvExtractionRuns.IgnoreQueryFilters().SingleAsync()).Status);
}
[Fact]
public async Task Deadline_before_claim_fails_the_dormant_extraction_run()
{
await using var fixture = await Fixture.CreateAsync();
await fixture.SeedUserAsync();
await using (var scope = fixture.Provider.CreateAsyncScope())
{
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Upload(File()));
await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().UserOperations
.ExecuteUpdateAsync(setters => setters.SetProperty(item => item.DeadlineAtUtc, DateTime.UtcNow.AddMinutes(-1)));
}
Assert.False(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
await using var verification = fixture.Provider.CreateAsyncScope();
var db = verification.ServiceProvider.GetRequiredService<JobTrackerContext>();
Assert.Equal(OperationStatuses.Failed, (await db.UserOperations.IgnoreQueryFilters().SingleAsync()).Status);
var run = await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync();
Assert.Equal("failed", run.Status);
Assert.Contains("deadline", run.ErrorMessage, StringComparison.OrdinalIgnoreCase);
Assert.NotNull(run.CompletedAtUtc);
}
private static ProfileCvController CreateController(IServiceProvider services)
{
var controller = services.GetRequiredService<ProfileCvController>();
@@ -249,6 +249,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
cancellationToken);
if (affected == 1 && !canRetry)
{
await SynchronizeCvRunAsync(operation, OperationStatuses.Failed, message, now, cancellationToken);
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Failed, now));
await db.SaveChangesAsync(cancellationToken);
}
@@ -272,6 +273,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
cancellationToken);
if (cancelled == 1)
{
await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
await db.SaveChangesAsync(cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
@@ -303,6 +305,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
cancellationToken);
if (affected == 1)
{
await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
await db.SaveChangesAsync(cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
@@ -332,6 +335,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
var existingNotification = await db.UserNotifications.FirstOrDefaultAsync(item => item.OperationId == operationId, cancellationToken);
if (existingNotification is not null) db.UserNotifications.Remove(existingNotification);
await db.SaveChangesAsync(cancellationToken);
await SynchronizeCvRunAsync(operation, OperationStatuses.Queued, null, UtcNow, cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
return true;
}
@@ -393,6 +397,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
.SetProperty(item => item.LeaseExpiresAtUtc, (DateTime?)null), cancellationToken);
if (affected == 1)
{
await SynchronizeCvRunAsync(operation, status, message, now, cancellationToken);
db.UserNotifications.Add(CreateTerminalNotification(operation, status, now));
await db.SaveChangesAsync(cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
@@ -405,6 +410,43 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
return await db.Database.BeginTransactionAsync(cancellationToken);
}
private Task<int> SynchronizeCvRunAsync(
UserOperation operation,
string operationStatus,
string? message,
DateTime now,
CancellationToken cancellationToken)
{
if (!string.Equals(operation.TaskType, CvProcessingQueue.TaskType, StringComparison.Ordinal)
|| !string.Equals(operation.SubjectType, CvProcessingQueue.SubjectType, StringComparison.Ordinal)
|| !int.TryParse(operation.SubjectId, System.Globalization.NumberStyles.None,
System.Globalization.CultureInfo.InvariantCulture, out var runId)
|| runId <= 0)
{
return Task.FromResult(0);
}
var runStatus = operationStatus switch
{
OperationStatuses.Cancelled => "cancelled",
OperationStatuses.Failed => "failed",
OperationStatuses.Queued => "queued",
_ => null,
};
if (runStatus is null) return Task.FromResult(0);
var completedAt = OperationStatuses.IsTerminal(operationStatus)
? new DateTimeOffset(DateTime.SpecifyKind(now, DateTimeKind.Utc))
: (DateTimeOffset?)null;
return db.CvExtractionRuns.IgnoreQueryFilters()
.Where(run => run.Id == runId && run.OwnerUserId == operation.OwnerUserId)
.ExecuteUpdateAsync(setters => setters
.SetProperty(run => run.Status, runStatus)
.SetProperty(run => run.ErrorMessage, message)
.SetProperty(run => run.CompletedAtUtc, completedAt),
cancellationToken);
}
private static UserNotification CreateTerminalNotification(UserOperation operation, string status, DateTime now)
{
var (kind, title, message) = status switch
+1
View File
@@ -211,3 +211,4 @@ Output was reduced to filenames and commit counts. The token artifact appears un
| V-177 | Python unit tests and plan-only execution against the checked-in synthetic fixture; script/content/SSRF/report review | Repository root | Prepare reproducible privacy-safe PROD-003 evaluation without authorizing a model or network call | PASS — harness tests 4/4; plan validates four Strategy cases across 4K/8K for eight future requests. Default performs no HTTP; exact model, `--execute` and explicit output are required; model pull/delete is absent; report excludes raw fixture/prompt/output content | No Ollama request, provider/internet access, model metadata query through the harness, candidate pull, inference or production change. Candidate metadata/licenses/results and model decision remain unmeasured | PROD-003 repository harness complete; execution remains blocked |
| V-178 | Benchmark harness safety tests, plan-only execution and CI workflow inspection | Repository root | Prevent an approved private Ollama origin from escaping through proxy settings or redirects and make the boundary a release gate | PASS — 5/5 standard-library tests; proxy discovery is disabled, redirects are refused, plan-only output remains eight future Strategy requests, and CI now runs the suite without dependencies or network execution | No network, Ollama, provider, package or production call occurred | Benchmark request boundary corrected and CI-enforced |
| V-179 | Account-deletion real-SQLite failure/retry tests; sidecar token/cache tests; full backend; Compose validation | Repository root / `tools/summarizer` | Remove the live sidecar-cache and shared tombstone-path gaps without enabling deletion | PASS — lifecycle 6/6, backend 658/658, sidecar 23/23 and Compose config pass. Sidecar failure withholds completion/tombstone until retry; maintenance purge is token-protected; tombstones map to a separate named volume; activation defaults false | Synthetic rows/cache only; no production volume, deletion, restart, provider revocation, backup restore or retention decision | SEC-009 repository cache/storage boundary complete; production activation remains blocked |
| V-180 | CV operation/store focused real-SQLite tests and full backend | Repository root | Keep dormant CV extraction history consistent with cancellation, deadline recovery and retry before worker claim | PASS — focused 17/17 and backend 660/660. Cancel sets the run terminal immediately, retry reopens it, deadline recovery fails it, and owner/task/subject predicates prevent unrelated updates | Synthetic rows only; no parser/model/MariaDB/production process interruption | AI-004 dormant-row consistency gap closed |
@@ -34,12 +34,14 @@ No dependency, schema, migration, proxy timeout or production switch changed. `W
## Remaining gates and known limits
- SEC-006/007 still own fixed parser versions, page/pixel/decompression/memory/process isolation and complete parser-child cancellation/cleanup. Legacy structured parsing calls are not all cancellation-aware. No malicious file was executed.
- Cancellation of an operation before a worker claims it is authoritative in `UserOperation` and shown correctly after refresh; the underlying extraction row is reconciled when retried/processed, but immediate terminal synchronization of that dormant row remains follow-up cleanup work.
- Operation cancellation, terminal failure/deadline recovery, and explicit retry now synchronize the referenced owner-scoped extraction row immediately. A queued CV operation can no longer leave history stuck at `queued` after it is cancelled or expires before worker claim.
- Browser localhost is denied by administrator policy. No real browser/mobile/theme/keyboard/refresh/back-forward workflow or screenshot is claimed.
- The authorized private CV was not used. Synthetic input must pass the SEC-006/007 gates before that local-only check.
- The generic lease tests cover restart recovery, but no CV parser/model process was interrupted and resumed in a runtime canary.
- MariaDB, selected Ollama model, worker telemetry, production activation and rollback canary remain unverified. The worker stays default-off.
The focused CV/operation/store regression slice is now 17/17 and the full backend is 660/660 after dormant-row cancellation/deadline/retry coverage.
## Rollback
Keep `Workers:AiOperationsEnabled=false`, revert `c3c5af8`, and retain the additive operation/extraction tables. Cancel or drain queued `cv.process` operations before removing the handler. No database downgrade or artifact rewrite is required; existing extraction runs remain readable.
+10
View File
@@ -779,3 +779,13 @@
- **Consequences:** a rare account deletion globally clears the shared summary cache because entries are not owner-keyed. An unavailable sidecar delays completion but does not restore deleted live rows/files. Production still needs protected volume custody, provider semantics, retention decisions and a disposable restore rehearsal.
- **User approval required:** Production activation and operational mutation only. Repository implementation remains inert behind `ACCOUNT_DELETION_ENABLED=false`.
- **Reversible:** Keep deletion disabled, drain any in-flight request, then revert the endpoint/client and Compose volume mapping. Never remove a deployed tombstone ledger while older restorable backups exist.
## DEC-079 — Synchronize durable CV terminal state centrally
- **Date:** 2026-08-15
- **Decision:** When the shared operation store cancels, terminally fails, recovers an expired deadline, or retries a `cv.process` operation, update its explicitly referenced owner-scoped `CvExtractionRun` in the same transaction boundary.
- **Reason/evidence:** cancellation before worker claim was authoritative in `UserOperation` but left the domain history row queued indefinitely. Focused 17/17 tests cover cancel-before-claim, retry, deadline recovery and existing lease behavior; backend 660/660 passes.
- **Alternatives considered:** reconcile only in the controller; wait for a later worker; add a generic observer framework. These miss recovery paths, preserve stale history, or overbuild a two-task operation system.
- **Consequences:** generic operations remain independent; the one existing persisted subject projection is synchronized through a narrow task/subject check with an explicit owner predicate.
- **User approval required:** No; local consistency fix with no schema, dependency or production change.
- **Reversible:** Revert the store helper/tests. No stored format changed.
+2 -2
View File
@@ -558,9 +558,9 @@ This queue records the highest-value work that can proceed without production cr
- **Required production verification:** synthetic/local-only canary, no external payload, restart recovery.
- **Status:** `IMPLEMENTED — NOT VERIFIED`.
- **Blocker:** SEC-006 dependency upgrades need internet permission; browser/private-file/MariaDB/production reproduction remains unavailable. Synthetic repository work can continue.
- **Evidence:** `docs/verification/ai-004-cv-processing-queue.md`; V-104V-107; real SQLite synthetic integration proves 202/active deduplication/owner-scoped handler/retry provenance/notification/review gate; backend 594/594; frontend 161/161 and build.
- **Evidence:** `docs/verification/ai-004-cv-processing-queue.md`; V-104V-107/V-180; real SQLite synthetic integration proves 202/active deduplication/owner-scoped handler/retry provenance/notification/review gate and pre-claim cancellation/deadline synchronization; focused operation lifecycle 17/17; backend 660/660; frontend 161/161 and build.
- **Commit:** `c3c5af8` (`feat(cv)!: queue durable processing`).
- **Remaining work:** SEC-006/007 parser dependency/isolation and complete parser cancellation; browser synthetic upload/refresh/retry/cancel/review at required widths/themes/keyboard; selected-model and worker-restart canary; MariaDB/production rollout. Reconcile dormant extraction-row status immediately when an operation is cancelled before claim. Do not use the private CV before safeguards.
- **Remaining work:** SEC-006/007 parser dependency/isolation and complete parser-child cancellation; browser synthetic upload/refresh/retry/cancel/review at required widths/themes/keyboard; selected-model and worker-restart canary; MariaDB/production rollout. Do not use the private CV before safeguards.
### UX-001 — Unified authentication page