cesnimda/jobtrackingapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(processing): persist safe atomic operation history

Browse Source
This commit is contained in:
cesnimda
2026-10-28 15:54:37 +01:00
parent 5f261083f3
commit 1f87301ee9
17 changed files with 501 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
JobTrackerApi/Services/UserOperationStore.cs
+83 -23
View File
@@ -75,6 +75,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
DeadlineAtUtc = request.DeadlineAtUtc,
};
db.UserOperations.Add(operation);
var createdEvent = OperationEventLog.Create(operation, OperationEventKind.Created, now);
db.UserOperationEvents.Add(createdEvent);
AiUsageRecord? usage = null;
if (request.UsageReservedTokens > 0)
{
@@ -95,6 +97,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
catch (DbUpdateException)
{
db.Entry(operation).State = EntityState.Detached;
db.Entry(createdEvent).State = EntityState.Detached;
if (usage is not null) db.Entry(usage).State = EntityState.Detached;
existing = await db.UserOperations.FirstOrDefaultAsync(
item => item.TaskType == request.TaskType && item.IdempotencyKey == request.IdempotencyKey,
@@ -140,6 +143,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
foreach (var candidateId in candidateIds)
{
var leaseToken = Guid.NewGuid().ToString("N");
await using var transaction = await BeginTransactionAsync(cancellationToken);
var affected = await db.UserOperations.IgnoreQueryFilters()
.Where(operation => operation.Id == candidateId &&
(operation.Status == OperationStatuses.Queued || operation.Status == OperationStatuses.WaitingForRetry) &&
@@ -158,20 +162,23 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
var claimed = await db.UserOperations.IgnoreQueryFilters().AsNoTracking()
.SingleAsync(operation => operation.Id == candidateId && operation.LeaseToken == leaseToken, cancellationToken);
await OperationEventLog.AppendAsync(db, claimed, OperationEventKind.Claimed, now, cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
return new UserOperationLease(claimed.Id, claimed.OwnerUserId, leaseToken, claimed.TaskType, claimed.PrivacyPolicy, claimed.SubjectType, claimed.SubjectId, claimed.AttemptCount, claimed.DeadlineAtUtc);
}
return null;
}
public Task<int> HeartbeatAsync(Guid operationId, string leaseToken, TimeSpan leaseDuration, string? progressStage, int? progressPercent, CancellationToken cancellationToken)
public async Task<int> HeartbeatAsync(Guid operationId, string leaseToken, TimeSpan leaseDuration, string? progressStage, int? progressPercent, CancellationToken cancellationToken)
{
EnsureOwnerScope();
ValidateLeaseDuration(leaseDuration);
if (progressPercent is < 0 or > 100) throw new ArgumentOutOfRangeException(nameof(progressPercent));
ValidateOptional(progressStage, 64, nameof(progressStage));
if (progressStage is not null && await ReportStageAsync(operationId, leaseToken, progressStage, cancellationToken) != 1) return 0;
var now = UtcNow;
return db.UserOperations
return await db.UserOperations
.Where(operation => operation.Id == operationId && operation.Status == OperationStatuses.Running && operation.LeaseToken == leaseToken)
.ExecuteUpdateAsync(setters => setters
.SetProperty(operation => operation.LeaseExpiresAtUtc, now.Add(leaseDuration))
@@ -181,6 +188,26 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
cancellationToken);
}
public async Task<int> ReportStageAsync(Guid operationId, string leaseToken, string stage, CancellationToken cancellationToken)
{
EnsureOwnerScope();
ValidateRequired(stage, 64, nameof(stage));
var now = UtcNow;
await using var transaction = await BeginTransactionAsync(cancellationToken);
var active = db.UserOperations.Where(item => item.Id == operationId && item.LeaseToken == leaseToken &&
item.Status == OperationStatuses.Running && item.CancellationRequestedAtUtc == null && item.LeaseExpiresAtUtc > now);
var updated = await active.Where(item => item.ProgressStage != stage)
.ExecuteUpdateAsync(setters => setters.SetProperty(item => item.ProgressStage, stage), cancellationToken);
if (updated == 1)
{
var operation = await active.AsNoTracking().SingleAsync(cancellationToken);
await OperationEventLog.AppendAsync(db, operation, OperationEventKind.StageChanged, now, cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
return 1;
}
return await active.AnyAsync(cancellationToken) ? 1 : 0;
}
public Task<int> CompleteAsync(Guid operationId, string leaseToken, string? resultReference, CancellationToken cancellationToken)
=> CompleteAsync(operationId, leaseToken, resultReference, null, null, null, cancellationToken);
@@ -240,6 +267,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
.SetProperty(item => item.EstimatedTokenCount, estimatedTokens), cancellationToken);
}
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Succeeded, now));
await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Completed, now, cancellationToken,
status: OperationStatuses.Succeeded, stage: completionStage ?? "");
await db.SaveChangesAsync(cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
}
@@ -296,7 +325,12 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Failed, now));
await db.SaveChangesAsync(cancellationToken);
}
if (affected == 1 && transaction is not null) await transaction.CommitAsync(cancellationToken);
if (affected == 1)
{
await OperationEventLog.AppendAsync(db, operation, canRetry ? OperationEventKind.RetryScheduled : OperationEventKind.Failed, now, cancellationToken,
status: canRetry ? OperationStatuses.WaitingForRetry : OperationStatuses.Failed, stage: progressStage ?? "", failure: category);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
}
return affected == 1;
}
@@ -318,6 +352,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
{
await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Cancelled, now, cancellationToken, status: OperationStatuses.Cancelled);
await db.SaveChangesAsync(cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
return true;
@@ -326,7 +361,11 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
var requested = await db.UserOperations
.Where(item => item.Id == operationId && item.Status == OperationStatuses.Running && item.CancellationRequestedAtUtc == null)
.ExecuteUpdateAsync(setters => setters.SetProperty(item => item.CancellationRequestedAtUtc, now), cancellationToken);
if (requested == 1 && transaction is not null) await transaction.CommitAsync(cancellationToken);
if (requested == 1)
{
await OperationEventLog.AppendAsync(db, operation, OperationEventKind.CancellationRequested, now, cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
}
return requested == 1;
}
@@ -350,6 +389,7 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
{
await SynchronizeCvRunAsync(operation, OperationStatuses.Cancelled, "CV processing was cancelled.", now, cancellationToken);
db.UserNotifications.Add(CreateTerminalNotification(operation, OperationStatuses.Cancelled, now));
await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Cancelled, now, cancellationToken, status: OperationStatuses.Cancelled);
await db.SaveChangesAsync(cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
}
@@ -360,26 +400,30 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
{
EnsureOwnerScope();
await using var transaction = await BeginTransactionAsync(cancellationToken);
var operation = await db.UserOperations.FirstOrDefaultAsync(item => item.Id == operationId, cancellationToken);
var operation = await db.UserOperations.AsNoTracking().FirstOrDefaultAsync(item => item.Id == operationId, cancellationToken);
if (operation is null || operation.Status is not (OperationStatuses.Failed or OperationStatuses.Cancelled)) return false;
if (operation.TaskType == CvProcessingQueue.TaskType && operation.SubjectType == CvProcessingQueue.SubjectType
&& await SynchronizeCvRunAsync(operation, OperationStatuses.Queued, null, UtcNow, cancellationToken) != 1) return false;
operation.Status = OperationStatuses.Queued;
operation.AttemptCount = 0;
operation.AvailableAtUtc = UtcNow;
operation.StartedAtUtc = null;
operation.CompletedAtUtc = null;
operation.CancellationRequestedAtUtc = null;
operation.FailureCategory = null;
operation.FailureMessage = null;
operation.ResultReference = null;
operation.Provider = null;
operation.Model = null;
operation.ProgressStage = null;
operation.ProgressPercent = null;
var existingNotification = await db.UserNotifications.FirstOrDefaultAsync(item => item.OperationId == operationId, cancellationToken);
if (existingNotification is not null) db.UserNotifications.Remove(existingNotification);
await db.SaveChangesAsync(cancellationToken);
var now = UtcNow;
var affected = await db.UserOperations.Where(item => item.Id == operationId && item.Status == operation.Status && item.CompletedAtUtc == operation.CompletedAtUtc)
.ExecuteUpdateAsync(setters => setters
.SetProperty(item => item.Status, OperationStatuses.Queued)
.SetProperty(item => item.AttemptCount, 0)
.SetProperty(item => item.AvailableAtUtc, now)
.SetProperty(item => item.StartedAtUtc, (DateTime?)null)
.SetProperty(item => item.CompletedAtUtc, (DateTime?)null)
.SetProperty(item => item.CancellationRequestedAtUtc, (DateTime?)null)
.SetProperty(item => item.FailureCategory, (string?)null)
.SetProperty(item => item.FailureMessage, (string?)null)
.SetProperty(item => item.ResultReference, (string?)null)
.SetProperty(item => item.Provider, (string?)null)
.SetProperty(item => item.Model, (string?)null)
.SetProperty(item => item.ProgressStage, (string?)null)
.SetProperty(item => item.ProgressPercent, (int?)null), cancellationToken);
if (affected != 1) return false;
await db.UserNotifications.Where(item => item.OperationId == operationId).ExecuteDeleteAsync(cancellationToken);
await OperationEventLog.AppendAsync(db, operation, OperationEventKind.Retried, now, cancellationToken,
status: OperationStatuses.Queued, stage: "", attempt: 0);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
return true;
}
@@ -408,11 +452,18 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
foreach (var operation in deadlineExpired)
await FinalizeRecoveredAsync(operation, RecoveryTerminal.QueuedDeadline, OperationStatuses.Failed, "deadline_exceeded", "The operation deadline elapsed before work could complete.", now, cancellationToken);
await db.UserOperations.IgnoreQueryFilters()
var recoverable = await db.UserOperations.IgnoreQueryFilters().AsNoTracking()
.Where(operation => operation.Status == OperationStatuses.Running && operation.LeaseExpiresAtUtc <= now &&
operation.CancellationRequestedAtUtc == null && operation.AttemptCount < operation.MaxAttempts &&
(operation.DeadlineAtUtc == null || operation.DeadlineAtUtc > now))
.ExecuteUpdateAsync(setters => setters
.ToListAsync(cancellationToken);
foreach (var operation in recoverable)
{
await using var transaction = await BeginTransactionAsync(cancellationToken);
var affected = await db.UserOperations.IgnoreQueryFilters().Where(item => item.Id == operation.Id &&
item.Status == OperationStatuses.Running && item.LeaseExpiresAtUtc <= now && item.CancellationRequestedAtUtc == null &&
item.AttemptCount < item.MaxAttempts && (item.DeadlineAtUtc == null || item.DeadlineAtUtc > now))
.ExecuteUpdateAsync(setters => setters
.SetProperty(operation => operation.Status, OperationStatuses.WaitingForRetry)
.SetProperty(operation => operation.AvailableAtUtc, now)
.SetProperty(operation => operation.FailureCategory, "lease_expired")
@@ -420,6 +471,13 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
.SetProperty(operation => operation.LeaseToken, (string?)null)
.SetProperty(operation => operation.LeaseExpiresAtUtc, (DateTime?)null),
cancellationToken);
if (affected == 1)
{
await OperationEventLog.AppendAsync(db, operation, OperationEventKind.LeaseRecovered, now, cancellationToken,
status: OperationStatuses.WaitingForRetry, failure: "lease_expired");
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
}
}
}
private async Task FinalizeRecoveredAsync(UserOperation operation, RecoveryTerminal reason, string status, string category, string message, DateTime now, CancellationToken cancellationToken)
@@ -443,6 +501,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
{
await SynchronizeCvRunAsync(operation, status, message, now, cancellationToken);
db.UserNotifications.Add(CreateTerminalNotification(operation, status, now));
await OperationEventLog.AppendAsync(db, operation, status == OperationStatuses.Cancelled ? OperationEventKind.Cancelled : OperationEventKind.Expired,
now, cancellationToken, status: status, failure: category);
await db.SaveChangesAsync(cancellationToken);
if (transaction is not null) await transaction.CommitAsync(cancellationToken);
}