feat(processing): persist safe atomic operation history

JobTrackerApi/Controllers/ProfileCvController.Import.cs

@@ -23,8 +23,8 @@ public sealed partial class ProfileCvController
             cancellationToken.ThrowIfCancellationRequested();
             if (context is null) return;
             var lease = context.Lease;
-            var updated = await _db.UserOperations.Where(o => o.Id == lease.OperationId && o.LeaseToken == lease.LeaseToken && o.Status == OperationStatuses.Running && o.CancellationRequestedAtUtc == null && o.LeaseExpiresAtUtc > DateTime.UtcNow)
-                .ExecuteUpdateAsync(s => s.SetProperty(o => o.ProgressStage, stage), cancellationToken);
+            var updated = await new UserOperationStore(_db, TimeProvider.System)
+                .ReportStageAsync(lease.OperationId, lease.LeaseToken, stage, cancellationToken);
             if (updated != 1) throw new OperationCanceledException("The document processing lease is no longer active.", cancellationToken);
         }
         await Stage("cv_validating_file");

JobTrackerApi/Data/JobTrackerContext.cs

@@ -66,6 +66,7 @@ namespace JobTrackerApi.Data
     public DbSet<SubmittedPackageAttachment> SubmittedPackageAttachments => Set<SubmittedPackageAttachment>();
     public DbSet<InterviewPrepItem> InterviewPrepItems => Set<InterviewPrepItem>();
     public DbSet<UserOperation> UserOperations => Set<UserOperation>();
+    public DbSet<UserOperationEvent> UserOperationEvents => Set<UserOperationEvent>();
     public DbSet<UserNotification> UserNotifications => Set<UserNotification>();
     public DbSet<EmailSendAttempt> EmailSendAttempts => Set<EmailSendAttempt>();
     public DbSet<EmailDraft> EmailDrafts => Set<EmailDraft>();
@@ -321,6 +322,16 @@ namespace JobTrackerApi.Data
         modelBuilder.Entity<UserOperation>()
             .HasIndex(x => new { x.Status, x.AvailableAtUtc, x.Priority });
 
+        modelBuilder.Entity<UserOperationEvent>().HasQueryFilter(x => CurrentUserId != null && x.OwnerUserId == CurrentUserId);
+        modelBuilder.Entity<UserOperationEvent>().Property(x => x.OwnerUserId).HasMaxLength(255);
+        modelBuilder.Entity<UserOperationEvent>().Property(x => x.Kind).HasMaxLength(40);
+        modelBuilder.Entity<UserOperationEvent>().Property(x => x.Status).HasMaxLength(32);
+        modelBuilder.Entity<UserOperationEvent>().Property(x => x.Stage).HasMaxLength(64);
+        modelBuilder.Entity<UserOperationEvent>().Property(x => x.FailureCode).HasMaxLength(64);
+        modelBuilder.Entity<UserOperationEvent>().HasIndex(x => new { x.OperationId, x.Id });
+        modelBuilder.Entity<UserOperationEvent>().HasIndex(x => x.AtUtc);
+        modelBuilder.Entity<UserOperationEvent>().HasOne<UserOperation>().WithMany().HasForeignKey(x => x.OperationId).OnDelete(DeleteBehavior.Cascade);
+
         modelBuilder.Entity<UserNotification>()
             .HasQueryFilter(x => CurrentUserId != null && x.OwnerUserId == CurrentUserId);
         modelBuilder.Entity<UserNotification>().Property(x => x.OwnerUserId).HasMaxLength(255);

JobTrackerApi/Migrations/20261028090000_AddUserOperationEvents.cs

@@ -0,0 +1,50 @@
+using JobTrackerApi.Data;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace JobTrackerApi.Migrations;
+
+[DbContext(typeof(JobTrackerContext))]
+[Migration("20261028090000_AddUserOperationEvents")]
+public sealed class AddUserOperationEvents : Migration
+{
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        if (ActiveProvider?.Contains("MySql", StringComparison.OrdinalIgnoreCase) == true)
+            migrationBuilder.Sql("""
+                CREATE TABLE `UserOperationEvents` (
+                    `Id` bigint NOT NULL AUTO_INCREMENT,
+                    `OperationId` char(36) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `OwnerUserId` varchar(255) NOT NULL,
+                    `AtUtc` datetime(6) NOT NULL,
+                    `Kind` varchar(40) NOT NULL,
+                    `Status` varchar(32) NOT NULL,
+                    `Stage` varchar(64) NULL,
+                    `Attempt` int NOT NULL,
+                    `FailureCode` varchar(64) NULL,
+                    CONSTRAINT `PK_UserOperationEvents` PRIMARY KEY (`Id`),
+                    CONSTRAINT `FK_UserOperationEvents_UserOperations_OperationId` FOREIGN KEY (`OperationId`) REFERENCES `UserOperations` (`Id`) ON DELETE CASCADE
+                ) CHARACTER SET=utf8mb4;
+                """);
+        else
+            migrationBuilder.CreateTable("UserOperationEvents", table => new
+            {
+                Id = table.Column<long>(type: "INTEGER", nullable: false).Annotation("Sqlite:Autoincrement", true),
+                OperationId = table.Column<Guid>(type: "TEXT", nullable: false),
+                OwnerUserId = table.Column<string>(type: "TEXT", maxLength: 255, nullable: false),
+                AtUtc = table.Column<DateTime>(type: "TEXT", nullable: false),
+                Kind = table.Column<string>(type: "TEXT", maxLength: 40, nullable: false),
+                Status = table.Column<string>(type: "TEXT", maxLength: 32, nullable: false),
+                Stage = table.Column<string>(type: "TEXT", maxLength: 64, nullable: true),
+                Attempt = table.Column<int>(type: "INTEGER", nullable: false),
+                FailureCode = table.Column<string>(type: "TEXT", maxLength: 64, nullable: true),
+            }, constraints: table =>
+            {
+                table.PrimaryKey("PK_UserOperationEvents", x => x.Id);
+                table.ForeignKey("FK_UserOperationEvents_UserOperations_OperationId", x => x.OperationId, "UserOperations", "Id", onDelete: ReferentialAction.Cascade);
+            });
+        migrationBuilder.CreateIndex("IX_UserOperationEvents_OperationId_Id", "UserOperationEvents", new[] { "OperationId", "Id" });
+        migrationBuilder.CreateIndex("IX_UserOperationEvents_AtUtc", "UserOperationEvents", "AtUtc");
+    }
+    protected override void Down(MigrationBuilder migrationBuilder) => migrationBuilder.DropTable("UserOperationEvents");
+}

JobTrackerApi/Migrations/JobTrackerContextModelSnapshot.cs

@@ -2265,6 +2265,24 @@ namespace JobTrackerApi.Migrations
                     b.ToTable("TwoFactorRecoveryCodes");
                 });
 
+            modelBuilder.Entity("JobTrackerApi.Models.UserOperationEvent", b =>
+                {
+                    b.Property<long>("Id").ValueGeneratedOnAdd().HasColumnType("INTEGER");
+                    b.Property<Guid>("OperationId").HasColumnType("TEXT");
+                    b.Property<string>("OwnerUserId").IsRequired().HasMaxLength(255).HasColumnType("TEXT");
+                    b.Property<DateTime>("AtUtc").HasColumnType("TEXT");
+                    b.Property<string>("Kind").IsRequired().HasMaxLength(40).HasColumnType("TEXT");
+                    b.Property<string>("Status").IsRequired().HasMaxLength(32).HasColumnType("TEXT");
+                    b.Property<string>("Stage").HasMaxLength(64).HasColumnType("TEXT");
+                    b.Property<int>("Attempt").HasColumnType("INTEGER");
+                    b.Property<string>("FailureCode").HasMaxLength(64).HasColumnType("TEXT");
+                    b.HasKey("Id");
+                    b.HasIndex("OperationId", "Id");
+                    b.HasIndex("AtUtc");
+                    b.ToTable("UserOperationEvents");
+                    b.HasOne("JobTrackerApi.Models.UserOperation", null).WithMany().HasForeignKey("OperationId").OnDelete(DeleteBehavior.Cascade).IsRequired();
+                });
+
             modelBuilder.Entity("JobTrackerApi.Models.UserNotification", b =>
                 {
                     b.Property<Guid>("Id")

JobTrackerApi/Models/UserOperationEvent.cs

@@ -0,0 +1,20 @@
+namespace JobTrackerApi.Models;
+
+public enum OperationEventKind
+{
+    Created, Claimed, StageChanged, Completed, RetryScheduled, Failed,
+    CancellationRequested, Cancelled, Retried, LeaseRecovered, Expired
+}
+
+public sealed class UserOperationEvent
+{
+    public long Id { get; set; }
+    public Guid OperationId { get; set; }
+    public string OwnerUserId { get; set; } = string.Empty;
+    public DateTime AtUtc { get; set; }
+    public string Kind { get; set; } = string.Empty;
+    public string Status { get; set; } = string.Empty;
+    public string? Stage { get; set; }
+    public int Attempt { get; set; }
+    public string? FailureCode { get; set; }
+}

JobTrackerApi/Services/AccountDataExportService.cs

@@ -248,6 +248,9 @@ public sealed class AccountDataExportService(
                     }).ToListAsync(cancellationToken);
                 var notifications = await db.UserNotifications.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId).OrderBy(item => item.CreatedAtUtc).ToListAsync(cancellationToken);
                 await AddJsonAsync("data/operations.json", new { Operations = operations, Notifications = notifications }, operations.Count + notifications.Count);
+                var operationEvents = await db.UserOperationEvents.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId)
+                    .OrderBy(item => item.Id).ToListAsync(cancellationToken);
+                await AddJsonAsync("data/operation-events.json", operationEvents, operationEvents.Count);
 
                 var gmail = await db.GmailConnections.IgnoreQueryFilters().AsNoTracking().Where(item => item.OwnerUserId == ownerUserId)
                     .Select(item => new { item.GmailAddress, item.Scope, item.ConnectedAt, item.LastSyncedAt, item.LastSyncAttemptedAt, item.LastSyncSucceededAt, item.LastSyncMode, item.LastSyncSource, item.LastSyncStatus, item.LastSyncError }).ToListAsync(cancellationToken);

JobTrackerApi/Services/AccountDeletionService.cs

@@ -264,6 +264,7 @@ public sealed class AccountDeletionService(
             deleted += await db.UserRuleSettings.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.UserNotifications.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.UserOperations.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
+            deleted += await db.UserOperationEvents.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.JobApplications.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.Jobs.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);
             deleted += await db.Companies.IgnoreQueryFilters().Where(item => item.OwnerUserId == owner).ExecuteDeleteAsync(cancellationToken);

JobTrackerApi/Services/CvImportRetentionHostedService.cs

@@ -22,6 +22,7 @@ public sealed class CvImportRetentionHostedService(IServiceScopeFactory scopes,
 
     public static async Task<int> SweepAsync(JobTrackerContext db, AppPaths paths, DateTimeOffset now, CancellationToken token)
     {
+        await OperationEventLog.PruneAsync(db, now.UtcDateTime, token);
         var runs = await db.CvExtractionRuns.IgnoreQueryFilters().Include(r => r.Artifact)
             .Where(r => r.ParserVersion == "docling-v1" && r.Artifact != null && r.Artifact.StoragePath != "")
             .OrderBy(r => r.Id).Take(1000).ToListAsync(token);
@@ -43,6 +44,7 @@ public sealed class CvImportRetentionHostedService(IServiceScopeFactory scopes,
                 operation.Status = Models.OperationStatuses.Cancelled;
                 operation.CancellationRequestedAtUtc = now.UtcDateTime;
                 operation.CompletedAtUtc = now.UtcDateTime;
+                db.UserOperationEvents.Add(OperationEventLog.Create(operation, Models.OperationEventKind.Cancelled, now.UtcDateTime));
             }
             await db.SaveChangesAsync(token);
             try

JobTrackerApi/Services/OperationEventLog.cs

@@ -0,0 +1,59 @@
+using JobTrackerApi.Data;
+using JobTrackerApi.Models;
+using Microsoft.EntityFrameworkCore;
+
+namespace JobTrackerApi.Services;
+
+public static class OperationEventLog
+{
+    public const int RetentionDays = 30;
+    public static string SafeStatus(string? value) => value is
+        OperationStatuses.Queued or OperationStatuses.Running or OperationStatuses.WaitingForRetry or OperationStatuses.WaitingForExternalFallback
+        or OperationStatuses.Succeeded or OperationStatuses.Failed or OperationStatuses.Cancelled ? value : "unknown";
+
+    public static string? SafeStage(string? value) => value is "cv_validating_file" or "cv_extracting" or "cv_identifying_sections"
+        or "cv_analysing" or "cv_validating_details" or "cv_preparing_review" or "cv_review_ready" ? value : null;
+
+    public static string? SafeFailure(string? value) => value is null ? null : value is
+        "timeout" or "lease_expired" or "deadline_exceeded" or "entitlement_changed" or "document_unavailable"
+        or "sidecar_unavailable" or "upstream_unavailable" or "invalid_response" or "circuit_open" or "rate_limited"
+        or "permission_denied" or "cancelled" or "upload_rejected" or "parser_failed" or "malformed_output" or "provider_unavailable"
+        ? value : "processing_failed";
+
+    public static UserOperationEvent Create(UserOperation operation, OperationEventKind kind, DateTime now,
+        string? status = null, string? stage = null, string? failure = null, int? attempt = null) => new()
+    {
+        OperationId = operation.Id, OwnerUserId = operation.OwnerUserId, AtUtc = now, Kind = kind.ToString(),
+        Status = SafeStatus(status ?? operation.Status), Stage = SafeStage(stage ?? operation.ProgressStage),
+        Attempt = Math.Clamp(attempt ?? operation.AttemptCount, 0, 10), FailureCode = SafeFailure(failure),
+    };
+
+    public static async Task AppendAsync(JobTrackerContext db, UserOperation operation, OperationEventKind kind, DateTime now,
+        CancellationToken token, string? status = null, string? stage = null, string? failure = null, int? attempt = null)
+    {
+        var item = Create(operation, kind, now, status, stage, failure, attempt);
+        if (!db.Database.IsRelational())
+        {
+            db.UserOperationEvents.Add(item);
+            await db.SaveChangesAsync(token);
+            return;
+        }
+        if (db.Database.CurrentTransaction is null) throw new InvalidOperationException("Operation history requires the state transition transaction.");
+        // Parameterized insert avoids flushing unrelated entities tracked by a caller.
+        await db.Database.ExecuteSqlInterpolatedAsync($"INSERT INTO `UserOperationEvents` (`OperationId`,`OwnerUserId`,`AtUtc`,`Kind`,`Status`,`Stage`,`Attempt`,`FailureCode`) VALUES ({item.OperationId},{item.OwnerUserId},{item.AtUtc},{item.Kind},{item.Status},{item.Stage},{item.Attempt},{item.FailureCode})", token);
+    }
+
+    public static async Task<int> PruneAsync(JobTrackerContext db, DateTime now, CancellationToken token)
+    {
+        var cutoff = now.AddDays(-RetentionDays);
+        var ids = await db.UserOperationEvents.IgnoreQueryFilters().Where(item => item.AtUtc < cutoff)
+            .OrderBy(item => item.Id).Select(item => item.Id).Take(5000).ToListAsync(token);
+        if (!db.Database.IsRelational())
+        {
+            var items = await db.UserOperationEvents.IgnoreQueryFilters().Where(item => ids.Contains(item.Id)).ToListAsync(token);
+            db.UserOperationEvents.RemoveRange(items);
+            return await db.SaveChangesAsync(token);
+        }
+        return await db.UserOperationEvents.IgnoreQueryFilters().Where(item => ids.Contains(item.Id)).ExecuteDeleteAsync(token);
+    }
+}

JobTrackerApi/Services/StartupSchemaOwnership.cs

@@ -61,6 +61,7 @@ internal static class StartupSchemaOwnership
         "TwoFactorRecoveryCodes",
         "UserNotifications",
         "UserOperations",
+        "UserOperationEvents",
         "UserRuleSettings",
         "UserSessions",
     };

JobTrackerApi/Services/UserOperationStore.cs

@@ -75,6 +75,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
             DeadlineAtUtc = request.DeadlineAtUtc,
         };
         db.UserOperations.Add(operation);
+        var createdEvent = OperationEventLog.Create(operation, OperationEventKind.Created, now);
+        db.UserOperationEvents.Add(createdEvent);
         AiUsageRecord? usage = null;
         if (request.UsageReservedTokens > 0)
         {
@@ -95,6 +97,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         catch (DbUpdateException)
         {
             db.Entry(operation).State = EntityState.Detached;
+            db.Entry(createdEvent).State = EntityState.Detached;
             if (usage is not null) db.Entry(usage).State = EntityState.Detached;
             existing = await db.UserOperations.FirstOrDefaultAsync(
                 item => item.TaskType == request.TaskType && item.IdempotencyKey == request.IdempotencyKey,
@@ -140,6 +143,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         foreach (var candidateId in candidateIds)
         {
             var leaseToken = Guid.NewGuid().ToString("N");
+            await using var transaction = await BeginTransactionAsync(cancellationToken);
             var affected = await db.UserOperations.IgnoreQueryFilters()
                 .Where(operation => operation.Id == candidateId &&
                     (operation.Status == OperationStatuses.Queued || operation.Status == OperationStatuses.WaitingForRetry) &&
@@ -158,20 +162,23 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
 
             var claimed = await db.UserOperations.IgnoreQueryFilters().AsNoTracking()
                 .SingleAsync(operation => operation.Id == candidateId && operation.LeaseToken == leaseToken, cancellationToken);
+            await OperationEventLog.AppendAsync(db, claimed, OperationEventKind.Claimed, now, cancellationToken);
+            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
             return new UserOperationLease(claimed.Id, claimed.OwnerUserId, leaseToken, claimed.TaskType, claimed.PrivacyPolicy, claimed.SubjectType, claimed.SubjectId, claimed.AttemptCount, claimed.DeadlineAtUtc);
         }
 
         return null;
     }
 
-    public Task<int> HeartbeatAsync(Guid operationId, string leaseToken, TimeSpan leaseDuration, string? progressStage, int? progressPercent, CancellationToken cancellationToken)
+    public async Task<int> HeartbeatAsync(Guid operationId, string leaseToken, TimeSpan leaseDuration, string? progressStage, int? progressPercent, CancellationToken cancellationToken)
     {
         EnsureOwnerScope();
         ValidateLeaseDuration(leaseDuration);
         if (progressPercent is < 0 or > 100) throw new ArgumentOutOfRangeException(nameof(progressPercent));
         ValidateOptional(progressStage, 64, nameof(progressStage));
+        if (progressStage is not null && await ReportStageAsync(operationId, leaseToken, progressStage, cancellationToken) != 1) return 0;
         var now = UtcNow;
-        return db.UserOperations
+        return await db.UserOperations
             .Where(operation => operation.Id == operationId && operation.Status == OperationStatuses.Running && operation.LeaseToken == leaseToken)
             .ExecuteUpdateAsync(setters => setters
                 .SetProperty(operation => operation.LeaseExpiresAtUtc, now.Add(leaseDuration))
@@ -181,6 +188,26 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
                 cancellationToken);
     }
 
+    public async Task<int> ReportStageAsync(Guid operationId, string leaseToken, string stage, CancellationToken cancellationToken)
+    {
+        EnsureOwnerScope();
+        ValidateRequired(stage, 64, nameof(stage));
+        var now = UtcNow;
+        await using var transaction = await BeginTransactionAsync(cancellationToken);
+        var active = db.UserOperations.Where(item => item.Id == operationId && item.LeaseToken == leaseToken &&
+            item.Status == OperationStatuses.Running && item.CancellationRequestedAtUtc == null && item.LeaseExpiresAtUtc > now);
+        var updated = await active.Where(item => item.ProgressStage != stage)
+            .ExecuteUpdateAsync(setters => setters.SetProperty(item => item.ProgressStage, stage), cancellationToken);
+        if (updated == 1)
+        {
+            var operation = await active.AsNoTracking().SingleAsync(cancellationToken);
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.StageChanged, now, cancellationToken);
+            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+            return 1;
+        }
+        return await active.AnyAsync(cancellationToken) ? 1 : 0;
+    }
+
     public Task<int> CompleteAsync(Guid operationId, string leaseToken, string? resultReference, CancellationToken cancellationToken)
         => CompleteAsync(operationId, leaseToken, resultReference, null, null, null, cancellationToken);
 
@@ -240,6 +267,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
                         .SetProperty(item => item.EstimatedTokenCount, estimatedTokens), cancellationToken);
             }
             db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Succeeded, now));
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Completed, now, cancellationToken,
+                status: OperationStatuses.Succeeded, stage: completionStage ?? "");
             await db.SaveChangesAsync(cancellationToken);
             if (transaction is not null) await transaction.CommitAsync(cancellationToken);
         }
@@ -296,7 +325,12 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
             db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Failed, now));
             await db.SaveChangesAsync(cancellationToken);
         }
-        if (affected == 1 && transaction is not null) await transaction.CommitAsync(cancellationToken);
+        if (affected == 1)
+        {
+            await OperationEventLog.AppendAsync(db, operation, canRetry ? OperationEventKind.RetryScheduled : OperationEventKind.Failed, now, cancellationToken,
+                status: canRetry ? OperationStatuses.WaitingForRetry : OperationStatuses.Failed, stage: progressStage ?? "", failure: category);
+            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+        }
         return affected == 1;
     }
 
@@ -318,6 +352,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         {
             await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
             db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Cancelled, now, cancellationToken, status: OperationStatuses.Cancelled);
             await db.SaveChangesAsync(cancellationToken);
             if (transaction is not null) await transaction.CommitAsync(cancellationToken);
             return true;
@@ -326,7 +361,11 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         var requested = await db.UserOperations
             .Where(item => item.Id == operationId && item.Status == OperationStatuses.Running && item.CancellationRequestedAtUtc == null)
             .ExecuteUpdateAsync(setters => setters.SetProperty(item => item.CancellationRequestedAtUtc, now), cancellationToken);
-        if (requested == 1 && transaction is not null) await transaction.CommitAsync(cancellationToken);
+        if (requested == 1)
+        {
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.CancellationRequested, now, cancellationToken);
+            if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+        }
         return requested == 1;
     }
 
@@ -350,6 +389,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         {
             await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
             db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
+            await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Cancelled, now, cancellationToken, status: OperationStatuses.Cancelled);
             await db.SaveChangesAsync(cancellationToken);
             if (transaction is not null) await transaction.CommitAsync(cancellationToken);
         }
@@ -360,26 +400,30 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
     {
         EnsureOwnerScope();
         await using var transaction = await BeginTransactionAsync(cancellationToken);
-        var operation = await db.UserOperations.FirstOrDefaultAsync(item => item.Id == operationId, cancellationToken);
+        var operation = await db.UserOperations.AsNoTracking().FirstOrDefaultAsync(item => item.Id == operationId, cancellationToken);
         if (operation is null || operation.Status is not (OperationStatuses.Failed or OperationStatuses.Cancelled)) return false;
         if (operation.TaskType == CvProcessingQueue.TaskType && operation.SubjectType == CvProcessingQueue.SubjectType
             && await SynchronizeCvRunAsync(operation, OperationStatuses.Queued, null, UtcNow, cancellationToken) != 1) return false;
-        operation.Status = OperationStatuses.Queued;
-        operation.AttemptCount = 0;
-        operation.AvailableAtUtc = UtcNow;
-        operation.StartedAtUtc = null;
-        operation.CompletedAtUtc = null;
-        operation.CancellationRequestedAtUtc = null;
-        operation.FailureCategory = null;
-        operation.FailureMessage = null;
-        operation.ResultReference = null;
-        operation.Provider = null;
-        operation.Model = null;
-        operation.ProgressStage = null;
-        operation.ProgressPercent = null;
-        var existingNotification = await db.UserNotifications.FirstOrDefaultAsync(item => item.OperationId == operationId, cancellationToken);
-        if (existingNotification is not null) db.UserNotifications.Remove(existingNotification);
-        await db.SaveChangesAsync(cancellationToken);
+        var now = UtcNow;
+        var affected = await db.UserOperations.Where(item => item.Id == operationId && item.Status == operation.Status && item.CompletedAtUtc == operation.CompletedAtUtc)
+            .ExecuteUpdateAsync(setters => setters
+                .SetProperty(item => item.Status, OperationStatuses.Queued)
+                .SetProperty(item => item.AttemptCount, 0)
+                .SetProperty(item => item.AvailableAtUtc, now)
+                .SetProperty(item => item.StartedAtUtc, (DateTime?)null)
+                .SetProperty(item => item.CompletedAtUtc, (DateTime?)null)
+                .SetProperty(item => item.CancellationRequestedAtUtc, (DateTime?)null)
+                .SetProperty(item => item.FailureCategory, (string?)null)
+                .SetProperty(item => item.FailureMessage, (string?)null)
+                .SetProperty(item => item.ResultReference, (string?)null)
+                .SetProperty(item => item.Provider, (string?)null)
+                .SetProperty(item => item.Model, (string?)null)
+                .SetProperty(item => item.ProgressStage, (string?)null)
+                .SetProperty(item => item.ProgressPercent, (int?)null), cancellationToken);
+        if (affected != 1) return false;
+        await db.UserNotifications.Where(item => item.OperationId == operationId).ExecuteDeleteAsync(cancellationToken);
+        await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Retried, now, cancellationToken,
+            status: OperationStatuses.Queued, stage: "", attempt: 0);
         if (transaction is not null) await transaction.CommitAsync(cancellationToken);
         return true;
     }
@@ -408,11 +452,18 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         foreach (var operation in deadlineExpired)
             await FinalizeRecoveredAsync(operation, RecoveryTerminal.QueuedDeadline, OperationStatuses.Failed, "deadline_exceeded", "The operation deadline elapsed before work could complete.", now, cancellationToken);
 
-        await db.UserOperations.IgnoreQueryFilters()
+        var recoverable = await db.UserOperations.IgnoreQueryFilters().AsNoTracking()
             .Where(operation => operation.Status == OperationStatuses.Running && operation.LeaseExpiresAtUtc <= now &&
                 operation.CancellationRequestedAtUtc == null && operation.AttemptCount < operation.MaxAttempts &&
                 (operation.DeadlineAtUtc == null || operation.DeadlineAtUtc > now))
-            .ExecuteUpdateAsync(setters => setters
+            .ToListAsync(cancellationToken);
+        foreach (var operation in recoverable)
+        {
+            await using var transaction = await BeginTransactionAsync(cancellationToken);
+            var affected = await db.UserOperations.IgnoreQueryFilters().Where(item => item.Id == operation.Id &&
+                item.Status == OperationStatuses.Running && item.LeaseExpiresAtUtc <= now && item.CancellationRequestedAtUtc == null &&
+                item.AttemptCount < item.MaxAttempts && (item.DeadlineAtUtc == null || item.DeadlineAtUtc > now))
+                .ExecuteUpdateAsync(setters => setters
                 .SetProperty(operation => operation.Status, OperationStatuses.WaitingForRetry)
                 .SetProperty(operation => operation.AvailableAtUtc, now)
                 .SetProperty(operation => operation.FailureCategory, "lease_expired")
@@ -420,6 +471,13 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
                 .SetProperty(operation => operation.LeaseToken, (string?)null)
                 .SetProperty(operation => operation.LeaseExpiresAtUtc, (DateTime?)null),
                 cancellationToken);
+            if (affected == 1)
+            {
+                await OperationEventLog.AppendAsync(db, operation, OperationEventKind.LeaseRecovered, now, cancellationToken,
+                    status: OperationStatuses.WaitingForRetry, failure: "lease_expired");
+                if (transaction is not null) await transaction.CommitAsync(cancellationToken);
+            }
+        }
     }
 
     private async Task FinalizeRecoveredAsync(UserOperation operation, RecoveryTerminal reason, string status, string category, string message, DateTime now, CancellationToken cancellationToken)
@@ -443,6 +501,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
         {
             await SynchronizeCvRunAsync(operation, status, message, now, cancellationToken);
             db.UserNotifications.Add(CreateTerminalNotification(operation, status, now));
+            await OperationEventLog.AppendAsync(db, operation, status == OperationStatuses.Cancelled ? OperationEventKind.Cancelled : OperationEventKind.Expired,
+                now, cancellationToken, status: status, failure: category);
             await db.SaveChangesAsync(cancellationToken);
             if (transaction is not null) await transaction.CommitAsync(cancellationToken);
         }