fix(cv): sync operation terminal state
This commit is contained in:
@@ -105,6 +105,71 @@ public sealed class CvProcessingOperationTests
|
||||
Assert.Null(run.CompletedAtUtc);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Cancellation_before_claim_and_retry_keep_extraction_history_in_sync()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
await fixture.SeedUserAsync();
|
||||
|
||||
Guid operationId;
|
||||
await using (var scope = fixture.Provider.CreateAsyncScope())
|
||||
{
|
||||
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
|
||||
Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Upload(File()));
|
||||
operationId = (await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().UserOperations.SingleAsync()).Id;
|
||||
}
|
||||
|
||||
await using (var scope = fixture.Provider.CreateAsyncScope())
|
||||
{
|
||||
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
|
||||
var store = scope.ServiceProvider.GetRequiredService<UserOperationStore>();
|
||||
Assert.True(await store.RequestCancellationAsync(operationId, default));
|
||||
|
||||
var cancelledRun = await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().CvExtractionRuns.AsNoTracking().SingleAsync();
|
||||
Assert.Equal("cancelled", cancelledRun.Status);
|
||||
Assert.NotNull(cancelledRun.CompletedAtUtc);
|
||||
}
|
||||
|
||||
await using (var scope = fixture.Provider.CreateAsyncScope())
|
||||
{
|
||||
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
|
||||
Assert.True(await scope.ServiceProvider.GetRequiredService<UserOperationStore>().RetryAsync(operationId, default));
|
||||
var queuedRun = await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().CvExtractionRuns.AsNoTracking().SingleAsync();
|
||||
Assert.Equal("queued", queuedRun.Status);
|
||||
Assert.Null(queuedRun.ErrorMessage);
|
||||
Assert.Null(queuedRun.CompletedAtUtc);
|
||||
}
|
||||
|
||||
Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
|
||||
await using var verification = fixture.Provider.CreateAsyncScope();
|
||||
Assert.Equal("pending_review", (await verification.ServiceProvider.GetRequiredService<JobTrackerContext>()
|
||||
.CvExtractionRuns.IgnoreQueryFilters().SingleAsync()).Status);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Deadline_before_claim_fails_the_dormant_extraction_run()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
await fixture.SeedUserAsync();
|
||||
|
||||
await using (var scope = fixture.Provider.CreateAsyncScope())
|
||||
{
|
||||
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
|
||||
Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Upload(File()));
|
||||
await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().UserOperations
|
||||
.ExecuteUpdateAsync(setters => setters.SetProperty(item => item.DeadlineAtUtc, DateTime.UtcNow.AddMinutes(-1)));
|
||||
}
|
||||
|
||||
Assert.False(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
|
||||
await using var verification = fixture.Provider.CreateAsyncScope();
|
||||
var db = verification.ServiceProvider.GetRequiredService<JobTrackerContext>();
|
||||
Assert.Equal(OperationStatuses.Failed, (await db.UserOperations.IgnoreQueryFilters().SingleAsync()).Status);
|
||||
var run = await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync();
|
||||
Assert.Equal("failed", run.Status);
|
||||
Assert.Contains("deadline", run.ErrorMessage, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.NotNull(run.CompletedAtUtc);
|
||||
}
|
||||
|
||||
private static ProfileCvController CreateController(IServiceProvider services)
|
||||
{
|
||||
var controller = services.GetRequiredService<ProfileCvController>();
|
||||
|
||||
@@ -249,6 +249,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
|
||||
cancellationToken);
|
||||
if (affected == 1 && !canRetry)
|
||||
{
|
||||
await SynchronizeCvRunAsync(operation, OperationStatuses.Failed, message, now, cancellationToken);
|
||||
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Failed, now));
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
@@ -272,6 +273,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
|
||||
cancellationToken);
|
||||
if (cancelled == 1)
|
||||
{
|
||||
await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
|
||||
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
|
||||
@@ -303,6 +305,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
|
||||
cancellationToken);
|
||||
if (affected == 1)
|
||||
{
|
||||
await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
|
||||
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
|
||||
@@ -332,6 +335,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
|
||||
var existingNotification = await db.UserNotifications.FirstOrDefaultAsync(item => item.OperationId == operationId, cancellationToken);
|
||||
if (existingNotification is not null) db.UserNotifications.Remove(existingNotification);
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
await SynchronizeCvRunAsync(operation, OperationStatuses.Queued, null, UtcNow, cancellationToken);
|
||||
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
|
||||
return true;
|
||||
}
|
||||
@@ -393,6 +397,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
|
||||
.SetProperty(item => item.LeaseExpiresAtUtc, (DateTime?)null), cancellationToken);
|
||||
if (affected == 1)
|
||||
{
|
||||
await SynchronizeCvRunAsync(operation, status, message, now, cancellationToken);
|
||||
db.UserNotifications.Add(CreateTerminalNotification(operation, status, now));
|
||||
await db.SaveChangesAsync(cancellationToken);
|
||||
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
|
||||
@@ -405,6 +410,43 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
|
||||
return await db.Database.BeginTransactionAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private Task<int> SynchronizeCvRunAsync(
|
||||
UserOperation operation,
|
||||
string operationStatus,
|
||||
string? message,
|
||||
DateTime now,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!string.Equals(operation.TaskType, CvProcessingQueue.TaskType, StringComparison.Ordinal)
|
||||
|| !string.Equals(operation.SubjectType, CvProcessingQueue.SubjectType, StringComparison.Ordinal)
|
||||
|| !int.TryParse(operation.SubjectId, System.Globalization.NumberStyles.None,
|
||||
System.Globalization.CultureInfo.InvariantCulture, out var runId)
|
||||
|| runId <= 0)
|
||||
{
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
var runStatus = operationStatus switch
|
||||
{
|
||||
OperationStatuses.Cancelled => "cancelled",
|
||||
OperationStatuses.Failed => "failed",
|
||||
OperationStatuses.Queued => "queued",
|
||||
_ => null,
|
||||
};
|
||||
if (runStatus is null) return Task.FromResult(0);
|
||||
var completedAt = OperationStatuses.IsTerminal(operationStatus)
|
||||
? new DateTimeOffset(DateTime.SpecifyKind(now, DateTimeKind.Utc))
|
||||
: (DateTimeOffset?)null;
|
||||
|
||||
return db.CvExtractionRuns.IgnoreQueryFilters()
|
||||
.Where(run => run.Id == runId && run.OwnerUserId == operation.OwnerUserId)
|
||||
.ExecuteUpdateAsync(setters => setters
|
||||
.SetProperty(run => run.Status, runStatus)
|
||||
.SetProperty(run => run.ErrorMessage, message)
|
||||
.SetProperty(run => run.CompletedAtUtc, completedAt),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
private static UserNotification CreateTerminalNotification(UserOperation operation, string status, DateTime now)
|
||||
{
|
||||
var (kind, title, message) = status switch
|
||||
|
||||
@@ -211,3 +211,4 @@ Output was reduced to filenames and commit counts. The token artifact appears un
|
||||
| V-177 | Python unit tests and plan-only execution against the checked-in synthetic fixture; script/content/SSRF/report review | Repository root | Prepare reproducible privacy-safe PROD-003 evaluation without authorizing a model or network call | PASS — harness tests 4/4; plan validates four Strategy cases across 4K/8K for eight future requests. Default performs no HTTP; exact model, `--execute` and explicit output are required; model pull/delete is absent; report excludes raw fixture/prompt/output content | No Ollama request, provider/internet access, model metadata query through the harness, candidate pull, inference or production change. Candidate metadata/licenses/results and model decision remain unmeasured | PROD-003 repository harness complete; execution remains blocked |
|
||||
| V-178 | Benchmark harness safety tests, plan-only execution and CI workflow inspection | Repository root | Prevent an approved private Ollama origin from escaping through proxy settings or redirects and make the boundary a release gate | PASS — 5/5 standard-library tests; proxy discovery is disabled, redirects are refused, plan-only output remains eight future Strategy requests, and CI now runs the suite without dependencies or network execution | No network, Ollama, provider, package or production call occurred | Benchmark request boundary corrected and CI-enforced |
|
||||
| V-179 | Account-deletion real-SQLite failure/retry tests; sidecar token/cache tests; full backend; Compose validation | Repository root / `tools/summarizer` | Remove the live sidecar-cache and shared tombstone-path gaps without enabling deletion | PASS — lifecycle 6/6, backend 658/658, sidecar 23/23 and Compose config pass. Sidecar failure withholds completion/tombstone until retry; maintenance purge is token-protected; tombstones map to a separate named volume; activation defaults false | Synthetic rows/cache only; no production volume, deletion, restart, provider revocation, backup restore or retention decision | SEC-009 repository cache/storage boundary complete; production activation remains blocked |
|
||||
| V-180 | CV operation/store focused real-SQLite tests and full backend | Repository root | Keep dormant CV extraction history consistent with cancellation, deadline recovery and retry before worker claim | PASS — focused 17/17 and backend 660/660. Cancel sets the run terminal immediately, retry reopens it, deadline recovery fails it, and owner/task/subject predicates prevent unrelated updates | Synthetic rows only; no parser/model/MariaDB/production process interruption | AI-004 dormant-row consistency gap closed |
|
||||
|
||||
@@ -34,12 +34,14 @@ No dependency, schema, migration, proxy timeout or production switch changed. `W
|
||||
## Remaining gates and known limits
|
||||
|
||||
- SEC-006/007 still own fixed parser versions, page/pixel/decompression/memory/process isolation and complete parser-child cancellation/cleanup. Legacy structured parsing calls are not all cancellation-aware. No malicious file was executed.
|
||||
- Cancellation of an operation before a worker claims it is authoritative in `UserOperation` and shown correctly after refresh; the underlying extraction row is reconciled when retried/processed, but immediate terminal synchronization of that dormant row remains follow-up cleanup work.
|
||||
- Operation cancellation, terminal failure/deadline recovery, and explicit retry now synchronize the referenced owner-scoped extraction row immediately. A queued CV operation can no longer leave history stuck at `queued` after it is cancelled or expires before worker claim.
|
||||
- Browser localhost is denied by administrator policy. No real browser/mobile/theme/keyboard/refresh/back-forward workflow or screenshot is claimed.
|
||||
- The authorized private CV was not used. Synthetic input must pass the SEC-006/007 gates before that local-only check.
|
||||
- The generic lease tests cover restart recovery, but no CV parser/model process was interrupted and resumed in a runtime canary.
|
||||
- MariaDB, selected Ollama model, worker telemetry, production activation and rollback canary remain unverified. The worker stays default-off.
|
||||
|
||||
The focused CV/operation/store regression slice is now 17/17 and the full backend is 660/660 after dormant-row cancellation/deadline/retry coverage.
|
||||
|
||||
## Rollback
|
||||
|
||||
Keep `Workers:AiOperationsEnabled=false`, revert `c3c5af8`, and retain the additive operation/extraction tables. Cancel or drain queued `cv.process` operations before removing the handler. No database downgrade or artifact rewrite is required; existing extraction runs remain readable.
|
||||
|
||||
@@ -779,3 +779,13 @@
|
||||
- **Consequences:** a rare account deletion globally clears the shared summary cache because entries are not owner-keyed. An unavailable sidecar delays completion but does not restore deleted live rows/files. Production still needs protected volume custody, provider semantics, retention decisions and a disposable restore rehearsal.
|
||||
- **User approval required:** Production activation and operational mutation only. Repository implementation remains inert behind `ACCOUNT_DELETION_ENABLED=false`.
|
||||
- **Reversible:** Keep deletion disabled, drain any in-flight request, then revert the endpoint/client and Compose volume mapping. Never remove a deployed tombstone ledger while older restorable backups exist.
|
||||
|
||||
## DEC-079 — Synchronize durable CV terminal state centrally
|
||||
|
||||
- **Date:** 2026-08-15
|
||||
- **Decision:** When the shared operation store cancels, terminally fails, recovers an expired deadline, or retries a `cv.process` operation, update its explicitly referenced owner-scoped `CvExtractionRun` in the same transaction boundary.
|
||||
- **Reason/evidence:** cancellation before worker claim was authoritative in `UserOperation` but left the domain history row queued indefinitely. Focused 17/17 tests cover cancel-before-claim, retry, deadline recovery and existing lease behavior; backend 660/660 passes.
|
||||
- **Alternatives considered:** reconcile only in the controller; wait for a later worker; add a generic observer framework. These miss recovery paths, preserve stale history, or overbuild a two-task operation system.
|
||||
- **Consequences:** generic operations remain independent; the one existing persisted subject projection is synchronized through a narrow task/subject check with an explicit owner predicate.
|
||||
- **User approval required:** No; local consistency fix with no schema, dependency or production change.
|
||||
- **Reversible:** Revert the store helper/tests. No stored format changed.
|
||||
|
||||
@@ -558,9 +558,9 @@ This queue records the highest-value work that can proceed without production cr
|
||||
- **Required production verification:** synthetic/local-only canary, no external payload, restart recovery.
|
||||
- **Status:** `IMPLEMENTED — NOT VERIFIED`.
|
||||
- **Blocker:** SEC-006 dependency upgrades need internet permission; browser/private-file/MariaDB/production reproduction remains unavailable. Synthetic repository work can continue.
|
||||
- **Evidence:** `docs/verification/ai-004-cv-processing-queue.md`; V-104–V-107; real SQLite synthetic integration proves 202/active deduplication/owner-scoped handler/retry provenance/notification/review gate; backend 594/594; frontend 161/161 and build.
|
||||
- **Evidence:** `docs/verification/ai-004-cv-processing-queue.md`; V-104–V-107/V-180; real SQLite synthetic integration proves 202/active deduplication/owner-scoped handler/retry provenance/notification/review gate and pre-claim cancellation/deadline synchronization; focused operation lifecycle 17/17; backend 660/660; frontend 161/161 and build.
|
||||
- **Commit:** `c3c5af8` (`feat(cv)!: queue durable processing`).
|
||||
- **Remaining work:** SEC-006/007 parser dependency/isolation and complete parser cancellation; browser synthetic upload/refresh/retry/cancel/review at required widths/themes/keyboard; selected-model and worker-restart canary; MariaDB/production rollout. Reconcile dormant extraction-row status immediately when an operation is cancelled before claim. Do not use the private CV before safeguards.
|
||||
- **Remaining work:** SEC-006/007 parser dependency/isolation and complete parser-child cancellation; browser synthetic upload/refresh/retry/cancel/review at required widths/themes/keyboard; selected-model and worker-restart canary; MariaDB/production rollout. Do not use the private CV before safeguards.
|
||||
|
||||
### UX-001 — Unified authentication page
|
||||
|
||||
|
||||
Reference in New Issue
Block a user