feat(processing): persist safe atomic operation history

JobTrackerApi/Services/AccountDataExportService.cs

@@ -248,6 +248,9 @@ public sealed class AccountDataExportService(
                     }).ToListAsync(cancellationToken);
                 var notifications = await db.UserNotifications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc).ToListAsync(cancellationToken);
                 await AddJsonAsync("data/operations.json", new { Operations = operations, Notifications = notifications }, operations.Count + notifications.Count);
+                var operationEvents = await db.UserOperationEvents.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId)
+                    .OrderBy(item => item.Id).ToListAsync(cancellationToken);
+                await AddJsonAsync("data/operation-events.json", operationEvents, operationEvents.Count);
 
                 var gmail = await db.GmailConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId)
                     .Select(item => new { item.GmailAddress, item.Scope, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken);

JobTrackerApi/Services/AccountDeletionService.cs

@@ -264,6 +264,7 @@ public sealed class AccountDeletionService(
             deleted += await db.UserRuleSettings.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.UserNotifications.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.UserOperations.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
+            deleted += await db.UserOperationEvents.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.JobApplications.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.Jobs.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.Companies.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);

JobTrackerApi/Services/CvImportRetentionHostedService.cs

@@ -22,6 +22,7 @@ public sealed class CvImportRetentionHostedService(IServiceScopeFactory scopes,
 
     public static async Task<int> SweepAsync(JobTrackerContext db, AppPaths paths, DateTimeOffset now, CancellationToken token)
     {
+        await OperationEventLog.PruneAsync(db, now.UtcDateTime, token);
         var runs = await db.CvExtractionRuns.IgnoreQueryFilters().Include(r => r.Artifact)
             .Where(r => r.ParserVersion == "docling-v1" && r.Artifact != null && r.Artifact.StoragePath != "")
             .OrderBy(r => r.Id).Take(1000).ToListAsync(token);
@@ -43,6 +44,7 @@ public sealed class CvImportRetentionHostedService(IServiceScopeFactory scopes,
                 operation.Status = Models.OperationStatuses.Cancelled;
                 operation.CancellationRequestedAtUtc = now.UtcDateTime;
                 operation.CompletedAtUtc = now.UtcDateTime;
+                db.UserOperationEvents.Add(OperationEventLog.Create(operation, Models.OperationEventKind.Cancelled, now.UtcDateTime));
             }
             await db.SaveChangesAsync(token);
             try

JobTrackerApi/Services/OperationEventLog.cs

@@ -0,0 +1,59 @@
+using JobTrackerApi.Data;
+using JobTrackerApi.Models;
+using Microsoft.EntityFrameworkCore;
+
+namespace JobTrackerApi.Services;
+
+public static class OperationEventLog
+{
+    public const int RetentionDays = 30;
+    public static string SafeStatus(string? value) => value is
+        OperationStatuses.Queued or OperationStatuses.Running or OperationStatuses.WaitingForRetry or OperationStatuses.WaitingForExternalFallback
+        or OperationStatuses.Succeeded or OperationStatuses.Failed or OperationStatuses.Cancelled ? value : "unknown";
+
+    public static string? SafeStage(string? value) => value is "cv_validating_file" or "cv_extracting" or "cv_identifying_sections"
+        or "cv_analysing" or "cv_validating_details" or "cv_preparing_review" or "cv_review_ready" ? value : null;
+
+    public static string? SafeFailure(string? value) => value is null ? null : value is
+        "timeout" or "lease_expired" or "deadline_exceeded" or "entitlement_changed" or "document_unavailable"
+        or "sidecar_unavailable" or "upstream_unavailable" or "invalid_response" or "circuit_open" or "rate_limited"
+        or "permission_denied" or "cancelled" or "upload_rejected" or "parser_failed" or "malformed_output" or "provider_unavailable"
+        ? value : "processing_failed";
+
+    public static UserOperationEvent Create(UserOperation operation, OperationEventKind kind, DateTime now,
+        string? status = null, string? stage = null, string? failure = null, int? attempt = null) => new()
+    {
+        OperationId = operation.Id, OwnerUserId = operation.OwnerUserId, AtUtc = now, Kind = kind.ToString(),
+        Status = SafeStatus(status ?? operation.Status), Stage = SafeStage(stage ?? operation.ProgressStage),
+        Attempt = Math.Clamp(attempt ?? operation.AttemptCount, 0, 10), FailureCode = SafeFailure(failure),
+    };
+
+    public static async Task AppendAsync(JobTrackerContext db, UserOperation operation, OperationEventKind kind, DateTime now,
+        CancellationToken token, string? status = null, string? stage = null, string? failure = null, int? attempt = null)
+    {
+        var item = Create(operation, kind, now, status, stage, failure, attempt);
+        if (!db.Database.IsRelational())
+        {
+            db.UserOperationEvents.Add(item);
+            await db.SaveChangesAsync(token);
+            return;
+        }
+        if (db.Database.CurrentTransaction is null) throw new InvalidOperationException("Operation history requires the state transition transaction.");
+        // Parameterized insert avoids flushing unrelated entities tracked by a caller.
+        await db.Database.ExecuteSqlInterpolatedAsync($"INSERT INTO `UserOperationEvents` (`OperationId`,`OwnerUserId`,`AtUtc`,`Kind`,`Status`,`Stage`,`Attempt`,`FailureCode`) VALUES ({item.OperationId},{item.OwnerUserId},{item.AtUtc},{item.Kind},{item.Status},{item.Stage},{item.Attempt},{item.FailureCode})", token);
+    }
+
+    public static async Task<int> PruneAsync(JobTrackerContext db, DateTime now, CancellationToken token)
+    {
+        var cutoff = now.AddDays(-RetentionDays);
+        var ids = await db.UserOperationEvents.IgnoreQueryFilters().Where(item => item.AtUtc < cutoff)
+            .OrderBy(item => item.Id).Select(item => item.Id).Take(5000).ToListAsync(token);
+        if (!db.Database.IsRelational())
+        {
+            var items = await db.UserOperationEvents.IgnoreQueryFilters().Where(item => ids.Contains(item.Id)).ToListAsync(token);
+            db.UserOperationEvents.RemoveRange(items);
+            return await db.SaveChangesAsync(token);
+        }
+        return await db.UserOperationEvents.IgnoreQueryFilters().Where(item => ids.Contains(item.Id)).ExecuteDeleteAsync(token);
+    }
+}

JobTrackerApi/Services/StartupSchemaOwnership.cs

@@ -61,6 +61,7 @@ internal static class StartupSchemaOwnership
         "TwoFactorRecoveryCodes",
         "UserNotifications",
         "UserOperations",
+        "UserOperationEvents",
         "UserRuleSettings",
         "UserSessions",
     };

JobTrackerApi/Services/UserOperationStore.cs

@@ -75,6 +75,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
             DeadlineAtUtc = request.DeadlineAtUtc,
         };
         db.UserOperations.Add(operation);
+        var createdEvent = OperationEventLog.Create(operation, OperationEventKind.Created, now);
+        db.UserOperationEvents.Add(createdEvent);
         AiUsageRecord? usage = null;
         if (request.UsageReservedTokens > 0)
         {
@@ -95,6 +97,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         catch (DbUpdateException)
         {
             db.Entry(operation).State = EntityState.Detached;
+            db.Entry(createdEvent).State = EntityState.Detached;
             if (usage is not null) db.Entry(usage).State = EntityState.Detached;
             existing = await db.UserOperations.FirstOrDefaultAsync(
                 item => item.TaskType == request.TaskType && item.IdempotencyKey == request.IdempotencyKey,
@@ -140,6 +143,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         foreach (var candidateId in candidateIds)
         {
             var leaseToken = Guid.NewGuid().ToString("N");
+            await using var transaction = await BeginTransactionAsync(cancellationToken);
             var affected = await db.UserOperations.IgnoreQueryFilters()
                 .Where(operation => operation.Id == candidateId &&
                     (operation.Status == OperationStatuses.Queued || operation.Status == OperationStatuses.WaitingForRetry) &&
@@ -158,20 +162,23 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
 
             var claimed = await db.UserOperations.IgnoreQueryFilters().AsNoTracking()
                 .SingleAsync(operation => operation.Id == candidateId && operation.LeaseToken == leaseToken, cancellationToken);
+            await OperationEventLog.AppendAsync(db, claimed, OperationEventKind.Claimed, now, cancellationToken);
+            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
             return new UserOperationLease(claimed.Id, claimed.OwnerUserId, leaseToken, claimed.TaskType, claimed.PrivacyPolicy, claimed.SubjectType, claimed.SubjectId, claimed.AttemptCount, claimed.DeadlineAtUtc);
         }
 
         return null;
     }
 
-    public Task<int> HeartbeatAsync(Guid operationId, string leaseToken, TimeSpan leaseDuration, string? progressStage, int? progressPercent, CancellationToken cancellationToken)
+    public async Task<int> HeartbeatAsync(Guid operationId, string leaseToken, TimeSpan leaseDuration, string? progressStage, int? progressPercent, CancellationToken cancellationToken)
     {
         EnsureOwnerScope();
         ValidateLeaseDuration(leaseDuration);
         if (progressPercent is < 0 or > 100) throw new ArgumentOutOfRangeException(nameof(progressPercent));
         ValidateOptional(progressStage, 64, nameof(progressStage));
+        if (progressStage is not null && await ReportStageAsync(operationId, leaseToken, progressStage, cancellationToken) != 1) return 0;
         var now = UtcNow;
-        return db.UserOperations
+        return await db.UserOperations
             .Where(operation => operation.Id == operationId && operation.Status == OperationStatuses.Running && operation.LeaseToken == leaseToken)
             .ExecuteUpdateAsync(setters => setters
                 .SetProperty(operation => operation.LeaseExpiresAtUtc, now.Add(leaseDuration))
@@ -181,6 +188,26 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
                 cancellationToken);
     }
 
+    public async Task<int> ReportStageAsync(Guid operationId, string leaseToken, string stage, CancellationToken cancellationToken)
+    {
+        EnsureOwnerScope();
+        ValidateRequired(stage, 64, nameof(stage));
+        var now = UtcNow;
+        await using var transaction = await BeginTransactionAsync(cancellationToken);
+        var active = db.UserOperations.Where(item => item.Id == operationId && item.LeaseToken == leaseToken &&
+            item.Status == OperationStatuses.Running && item.CancellationRequestedAtUtc == null && item.LeaseExpiresAtUtc > now);
+        var updated = await active.Where(item => item.ProgressStage != stage)
+            .ExecuteUpdateAsync(setters => setters.SetProperty(item => item.ProgressStage, stage), cancellationToken);
+        if (updated == 1)
+        {
+            var operation = await active.AsNoTracking().SingleAsync(cancellationToken);
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.StageChanged, now, cancellationToken);
+            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+            return 1;
+        }
+        return await active.AnyAsync(cancellationToken) ? 1 : 0;
+    }
+
     public Task<int> CompleteAsync(Guid operationId, string leaseToken, string? resultReference, CancellationToken cancellationToken)
         => CompleteAsync(operationId, leaseToken, resultReference, null, null, null, cancellationToken);
 
@@ -240,6 +267,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
                         .SetProperty(item => item.EstimatedTokenCount, estimatedTokens), cancellationToken);
             }
             db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Succeeded, now));
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Completed, now, cancellationToken,
+                status: OperationStatuses.Succeeded, stage: completionStage ?? "");
             await db.SaveChangesAsync(cancellationToken);
             if (transaction is not null) await transaction.CommitAsync(cancellationToken);
         }
@@ -296,7 +325,12 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
             db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Failed, now));
             await db.SaveChangesAsync(cancellationToken);
         }
-        if (affected == 1 && transaction is not null) await transaction.CommitAsync(cancellationToken);
+        if (affected == 1)
+        {
+            await OperationEventLog.AppendAsync(db, operation, canRetry ? OperationEventKind.RetryScheduled : OperationEventKind.Failed, now, cancellationToken,
+                status: canRetry ? OperationStatuses.WaitingForRetry : OperationStatuses.Failed, stage: progressStage ?? "", failure: category);
+            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+        }
         return affected == 1;
     }
 
@@ -318,6 +352,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         {
             await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
             db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Cancelled, now, cancellationToken, status: OperationStatuses.Cancelled);
             await db.SaveChangesAsync(cancellationToken);
             if (transaction is not null) await transaction.CommitAsync(cancellationToken);
             return true;
@@ -326,7 +361,11 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         var requested = await db.UserOperations
             .Where(item => item.Id == operationId && item.Status == OperationStatuses.Running && item.CancellationRequestedAtUtc == null)
             .ExecuteUpdateAsync(setters => setters.SetProperty(item => item.CancellationRequestedAtUtc, now), cancellationToken);
-        if (requested == 1 && transaction is not null) await transaction.CommitAsync(cancellationToken);
+        if (requested == 1)
+        {
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.CancellationRequested, now, cancellationToken);
+            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+        }
         return requested == 1;
     }
 
@@ -350,6 +389,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         {
             await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
             db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Cancelled, now, cancellationToken, status: OperationStatuses.Cancelled);
             await db.SaveChangesAsync(cancellationToken);
             if (transaction is not null) await transaction.CommitAsync(cancellationToken);
         }
@@ -360,26 +400,30 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
     {
         EnsureOwnerScope();
         await using var transaction = await BeginTransactionAsync(cancellationToken);
-        var operation = await db.UserOperations.FirstOrDefaultAsync(item => item.Id == operationId, cancellationToken);
+        var operation = await db.UserOperations.AsNoTracking().FirstOrDefaultAsync(item => item.Id == operationId, cancellationToken);
         if (operation is null || operation.Status is not (OperationStatuses.Failed or OperationStatuses.Cancelled)) return false;
         if (operation.TaskType == CvProcessingQueue.TaskType && operation.SubjectType == CvProcessingQueue.SubjectType
             && await SynchronizeCvRunAsync(operation, OperationStatuses.Queued, null, UtcNow, cancellationToken) != 1) return false;
-        operation.Status = OperationStatuses.Queued;
-        operation.AttemptCount = 0;
-        operation.AvailableAtUtc = UtcNow;
-        operation.StartedAtUtc = null;
-        operation.CompletedAtUtc = null;
-        operation.CancellationRequestedAtUtc = null;
-        operation.FailureCategory = null;
-        operation.FailureMessage = null;
-        operation.ResultReference = null;
-        operation.Provider = null;
-        operation.Model = null;
-        operation.ProgressStage = null;
-        operation.ProgressPercent = null;
-        var existingNotification = await db.UserNotifications.FirstOrDefaultAsync(item => item.OperationId == operationId, cancellationToken);
-        if (existingNotification is not null) db.UserNotifications.Remove(existingNotification);
-        await db.SaveChangesAsync(cancellationToken);
+        var now = UtcNow;
+        var affected = await db.UserOperations.Where(item => item.Id == operationId && item.Status == operation.Status && item.CompletedAtUtc == operation.CompletedAtUtc)
+            .ExecuteUpdateAsync(setters => setters
+                .SetProperty(item => item.Status, OperationStatuses.Queued)
+                .SetProperty(item => item.AttemptCount, 0)
+                .SetProperty(item => item.AvailableAtUtc, now)
+                .SetProperty(item => item.StartedAtUtc, (DateTime?)null)
+                .SetProperty(item => item.CompletedAtUtc, (DateTime?)null)
+                .SetProperty(item => item.CancellationRequestedAtUtc, (DateTime?)null)
+                .SetProperty(item => item.FailureCategory, (string?)null)
+                .SetProperty(item => item.FailureMessage, (string?)null)
+                .SetProperty(item => item.ResultReference, (string?)null)
+                .SetProperty(item => item.Provider, (string?)null)
+                .SetProperty(item => item.Model, (string?)null)
+                .SetProperty(item => item.ProgressStage, (string?)null)
+                .SetProperty(item => item.ProgressPercent, (int?)null), cancellationToken);
+        if (affected != 1) return false;
+        await db.UserNotifications.Where(item => item.OperationId == operationId).ExecuteDeleteAsync(cancellationToken);
+        await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Retried, now, cancellationToken,
+            status: OperationStatuses.Queued, stage: "", attempt: 0);
         if (transaction is not null) await transaction.CommitAsync(cancellationToken);
         return true;
     }
@@ -408,11 +452,18 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         foreach (var operation in deadlineExpired)
             await FinalizeRecoveredAsync(operation, RecoveryTerminal.QueuedDeadline, OperationStatuses.Failed, "deadline_exceeded", "The operation deadline elapsed before work could complete.", now, cancellationToken);
 
-        await db.UserOperations.IgnoreQueryFilters()
+        var recoverable = await db.UserOperations.IgnoreQueryFilters().AsNoTracking()
             .Where(operation => operation.Status == OperationStatuses.Running && operation.LeaseExpiresAtUtc <= now &&
                 operation.CancellationRequestedAtUtc == null && operation.AttemptCount < operation.MaxAttempts &&
                 (operation.DeadlineAtUtc == null || operation.DeadlineAtUtc > now))
-            .ExecuteUpdateAsync(setters => setters
+            .ToListAsync(cancellationToken);
+        foreach (var operation in recoverable)
+        {
+            await using var transaction = await BeginTransactionAsync(cancellationToken);
+            var affected = await db.UserOperations.IgnoreQueryFilters().Where(item => item.Id == operation.Id &&
+                item.Status == OperationStatuses.Running && item.LeaseExpiresAtUtc <= now && item.CancellationRequestedAtUtc == null &&
+                item.AttemptCount < item.MaxAttempts && (item.DeadlineAtUtc == null || item.DeadlineAtUtc > now))
+                .ExecuteUpdateAsync(setters => setters
                 .SetProperty(operation => operation.Status, OperationStatuses.WaitingForRetry)
                 .SetProperty(operation => operation.AvailableAtUtc, now)
                 .SetProperty(operation => operation.FailureCategory, "lease_expired")
@@ -420,6 +471,13 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
                 .SetProperty(operation => operation.LeaseToken, (string?)null)
                 .SetProperty(operation => operation.LeaseExpiresAtUtc, (DateTime?)null),
                 cancellationToken);
+            if (affected == 1)
+            {
+                await OperationEventLog.AppendAsync(db, operation, OperationEventKind.LeaseRecovered, now, cancellationToken,
+                    status: OperationStatuses.WaitingForRetry, failure: "lease_expired");
+                if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+            }
+        }
     }
 
     private async Task FinalizeRecoveredAsync(UserOperation operation, RecoveryTerminal reason, string status, string category, string message, DateTime now, CancellationToken cancellationToken)
@@ -443,6 +501,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         {
             await SynchronizeCvRunAsync(operation, status, message, now, cancellationToken);
             db.UserNotifications.Add(CreateTerminalNotification(operation, status, now));
+            await OperationEventLog.AppendAsync(db, operation, status == OperationStatuses.Cancelled ? OperationEventKind.Cancelled : OperationEventKind.Expired,
+                now, cancellationToken, status: status, failure: category);
             await db.SaveChangesAsync(cancellationToken);
             if (transaction is not null) await transaction.CommitAsync(cancellationToken);
         }