cesnimda/jobtrackingapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(admin): complete processing diagnostics and failure evidence

Browse Source
This commit is contained in:
cesnimda
2026-10-29 07:24:49 +01:00
parent 0dd788bdfc
commit 3255da9fc6
15 changed files with 450 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
JobTrackerApi/Controllers/AdminProcessingController.cs
+69 -8
View File
@@ -1,10 +1,12 @@
using System.Globalization;
using System.Security.Claims;
using System.Text.Json;
using JobTrackerApi.Data;
using JobTrackerApi.Models;
using JobTrackerApi.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.RateLimiting;
using Microsoft.EntityFrameworkCore;
namespace JobTrackerApi.Controllers;
@@ -12,15 +14,18 @@ namespace JobTrackerApi.Controllers;
[ApiController]
[Route("api/admin/processing")]
[Authorize(Roles = "Admin")]
[EnableRateLimiting(RateLimitPolicy)]
public sealed class AdminProcessingController(JobTrackerContext db, IConfiguration configuration, TimeProvider clock) : ControllerBase
{
public const string RateLimitPolicy = "admin-processing";
private static readonly HashSet<string> EventKinds = Enum.GetNames<OperationEventKind>().ToHashSet(StringComparer.Ordinal);
public sealed record IdentityDto(string Id, string? Name, string? Email);
public sealed record JobDto(int Id, string Title, string? Company);
public sealed record DocumentDto(string Name, string? Type, long Bytes, int? Pages, string? State);
public sealed record JobDto(int Id, string Title, string? Company, bool CanOpen);
public sealed record DocumentDto(int ImportId, string Name, string? Type, long Bytes, int? Pages, string? State);
public sealed record ItemDto(Guid Id, IdentityDto Owner, string TaskType, string Status, string? Stage, int AttemptCount,
int MaxAttempts, DateTime CreatedAtUtc, DateTime? StartedAtUtc, DateTime? CompletedAtUtc, DateTime? LastHeartbeatAtUtc,
double DurationSeconds, string Worker, bool WorkerEnabled, string? FailureCode, JobDto? Job, DocumentDto? Document);
double DurationSeconds, string Worker, bool WorkerEnabled, string? FailureCode, JobDto? Job, DocumentDto? Document,
DateTime? LastFailureAtUtc, string? LastFailureStage);
public sealed record WorkerDto(string Name, bool FlagEnabled, bool IndividualOperationsRecorded);
public sealed record PageDto(IReadOnlyList<ItemDto> Items, int Total, int Page, int PageSize,
IReadOnlyDictionary<string, int> Counts, IReadOnlyList<WorkerDto> Workers, string CvInterpretation);
@@ -29,17 +34,64 @@ public sealed class AdminProcessingController(JobTrackerContext db, IConfigurati
[HttpGet]
public async Task<ActionResult<PageDto>> List([FromQuery] string? status = null, [FromQuery] string? taskType = null,
[FromQuery] string? owner = null, [FromQuery] int page = 1, [FromQuery] int pageSize = 25, CancellationToken cancellationToken = default)
[FromQuery] string? owner = null, [FromQuery] int page = 1, [FromQuery] int pageSize = 25,
[FromQuery] int? jobId = null, [FromQuery] string? worker = null, [FromQuery] string? failureCode = null,
[FromQuery] string? q = null, [FromQuery] DateTimeOffset? createdFrom = null, [FromQuery] DateTimeOffset? createdBefore = null,
CancellationToken cancellationToken = default)
{
if (page is < 1 or > 1000 || pageSize is < 1 or > 100 || owner?.Length > 255
|| (status is not null && OperationEventLog.SafeStatus(status) == "unknown")
|| (taskType is not null && taskType is not (CvProcessingQueue.TaskType or StrategySnapshotService.TaskType or "other"))) return BadRequest();
|| (taskType is not null && taskType is not (CvProcessingQueue.TaskType or StrategySnapshotService.TaskType or "other"))
|| jobId <= 0 || q?.Length > 200 || (worker is not null && worker is not ("document" or "ai" or "untracked"))
|| (failureCode is not null && failureCode != "processing_failed" && OperationEventLog.SafeFailure(failureCode) != failureCode)
|| createdFrom?.UtcDateTime.Year < 1000 || createdBefore?.UtcDateTime.Year < 1000
|| (createdFrom is not null && createdBefore is not null && createdFrom >= createdBefore)) return BadRequest();
var all = ReadOperations();
var query = all;
// Historical tables use different MariaDB collations; owner identifiers must compare exactly.
var identifiers = db.Database.IsMySql() ? "utf8mb4_bin" : "BINARY";
if (status is not null) query = query.Where(item => item.Status == status);
if (!string.IsNullOrWhiteSpace(owner)) query = query.Where(item => item.OwnerUserId == owner);
if (taskType == "other") query = query.Where(item => item.TaskType != CvProcessingQueue.TaskType && item.TaskType != StrategySnapshotService.TaskType);
else if (taskType is not null) query = query.Where(item => item.TaskType == taskType);
if (createdFrom is not null) { var from = createdFrom.Value.UtcDateTime; query = query.Where(item => item.CreatedAtUtc >= from); }
if (createdBefore is not null) { var before = createdBefore.Value.UtcDateTime; query = query.Where(item => item.CreatedAtUtc < before); }
if (jobId is not null)
{
var prefix = jobId.Value.ToString(CultureInfo.InvariantCulture) + "|";
query = query.Where(item => item.TaskType == StrategySnapshotService.TaskType && item.SubjectType == "job_strategy"
&& item.SubjectId != null && item.SubjectId.StartsWith(prefix)
&& db.JobApplications.IgnoreQueryFilters().Any(job => job.Id == jobId && EF.Functions.Collate(job.OwnerUserId, identifiers) == item.OwnerUserId));
}
if (worker is not null)
{
var documents = db.CvExtractionRuns.IgnoreQueryFilters().Where(run => run.ParserVersion == ProfileCvController.DocumentParserVersion && run.Trigger == "document_import");
if (worker == "untracked") query = query.Where(item => item.TaskType != CvProcessingQueue.TaskType && item.TaskType != StrategySnapshotService.TaskType);
else if (worker == "document") query = query.Where(item => item.TaskType == CvProcessingQueue.TaskType && item.SubjectType == CvProcessingQueue.SubjectType
&& documents.Any(run => EF.Functions.Collate(run.OwnerUserId, identifiers) == item.OwnerUserId && run.Id.ToString() == item.SubjectId));
else query = query.Where(item => (item.TaskType == CvProcessingQueue.TaskType || item.TaskType == StrategySnapshotService.TaskType)
&& !(item.TaskType == CvProcessingQueue.TaskType && item.SubjectType == CvProcessingQueue.SubjectType
&& documents.Any(run => EF.Functions.Collate(run.OwnerUserId, identifiers) == item.OwnerUserId && run.Id.ToString() == item.SubjectId)));
}
if (failureCode is not null)
{
var known = OperationEventLog.FailureCodes.ToArray();
query = failureCode == "processing_failed"
? query.Where(item => item.FailureCategory != null && !known.Contains(item.FailureCategory))
: query.Where(item => item.FailureCategory == failureCode);
}
if (!string.IsNullOrWhiteSpace(q))
{
q = q.Trim();
var users = db.Users.Where(user => user.Id.Contains(q) || (user.DisplayName != null && user.DisplayName.Contains(q))
|| (user.Email != null && user.Email.Contains(q))).Select(user => user.Id);
var jobs = db.JobApplications.IgnoreQueryFilters().Where(job => job.JobTitle.Contains(q)
|| (EF.Functions.Collate(job.Company.OwnerUserId, identifiers) == job.OwnerUserId && job.Company.Name.Contains(q)));
var isId = Guid.TryParse(q, out var operationId);
query = query.Where(item => users.Any(id => EF.Functions.Collate(id, identifiers) == item.OwnerUserId) || (isId && item.Id == operationId)
|| (item.TaskType == StrategySnapshotService.TaskType && item.SubjectType == "job_strategy" && item.SubjectId != null
&& jobs.Any(job => EF.Functions.Collate(job.OwnerUserId, identifiers) == item.OwnerUserId && item.SubjectId.StartsWith(job.Id.ToString() + "|"))));
}
var total = await query.CountAsync(cancellationToken);
var operations = await query.OrderByDescending(item => item.CreatedAtUtc).ThenByDescending(item => item.Id)
.Skip((page - 1) * pageSize).Take(pageSize).ToListAsync(cancellationToken);
@@ -127,6 +179,13 @@ public sealed class AdminProcessingController(JobTrackerContext db, IConfigurati
private async Task<IReadOnlyList<ItemDto>> Project(IReadOnlyList<UserOperation> operations, bool includePages, CancellationToken token)
{
var selected = includePages && operations.Count == 1 ? operations[0] : null;
var code = OperationEventLog.SafeFailure(selected?.FailureCategory);
var cutoff = clock.GetUtcNow().UtcDateTime.AddDays(-OperationEventLog.RetentionDays);
var lastFailure = selected is null || code is null ? null : await db.UserOperationEvents.IgnoreQueryFilters().AsNoTracking()
.Where(item => item.OperationId == selected.Id && item.OwnerUserId == selected.OwnerUserId && item.AtUtc >= cutoff && item.FailureCode == code
&& (item.Kind == "Failed" || item.Kind == "RetryScheduled" || item.Kind == "LeaseRecovered" || item.Kind == "Expired"))
.OrderByDescending(item => item.Id).Select(item => new { item.AtUtc, item.Stage }).FirstOrDefaultAsync(token);
var ownerIds = operations.Select(item => item.OwnerUserId).Distinct().ToArray();
var users = await db.Users.AsNoTracking().Where(item => ownerIds.Contains(item.Id))
.Select(item => new { item.Id, Name = item.DisplayName ?? item.UserName, item.Email })
@@ -146,9 +205,9 @@ public sealed class AdminProcessingController(JobTrackerContext db, IConfigurati
{
var subject = Subject(operation);
var run = operation.TaskType == CvProcessingQueue.TaskType && subject is int cvId && runs.TryGetValue(cvId, out var foundRun) && foundRun.OwnerUserId == operation.OwnerUserId ? foundRun : null;
var job = operation.TaskType == StrategySnapshotService.TaskType && subject is int jobId && jobs.TryGetValue(jobId, out var foundJob) && foundJob.OwnerUserId == operation.OwnerUserId ? new JobDto(foundJob.Id, foundJob.JobTitle, foundJob.Company) : null;
var job = operation.TaskType == StrategySnapshotService.TaskType && subject is int jobId && jobs.TryGetValue(jobId, out var foundJob) && foundJob.OwnerUserId == operation.OwnerUserId ? new JobDto(foundJob.Id, foundJob.JobTitle, foundJob.Company, foundJob.OwnerUserId == HttpContext?.User.FindFirstValue(ClaimTypes.NameIdentifier)) : null;
var artifact = run?.ArtifactId is int artifactId && artifacts.TryGetValue(artifactId, out var foundArtifact) && foundArtifact.OwnerUserId == operation.OwnerUserId ? foundArtifact : null;
var document = artifact is null ? null : new DocumentDto(FileName(artifact.OriginalFileName),
var document = artifact is null ? null : new DocumentDto(run!.Id, FileName(artifact.OriginalFileName),
artifact.MimeType is "application/pdf" or "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ? artifact.MimeType : null,
Math.Max(0, artifact.ByteSize), Pages(run?.DocumentJson), run?.Status is "pending_review" or "applied" or "expired" or "failed" or "cancelled" or "running" or "queued" ? run.Status : null);
var worker = SafeTask(operation.TaskType) == "other" ? "untracked" : run is { ParserVersion: ProfileCvController.DocumentParserVersion, Trigger: "document_import" } ? "document" : "ai";
@@ -156,7 +215,9 @@ public sealed class AdminProcessingController(JobTrackerContext db, IConfigurati
return new ItemDto(operation.Id, users.GetValueOrDefault(operation.OwnerUserId) ?? new(operation.OwnerUserId, null, null), SafeTask(operation.TaskType),
OperationEventLog.SafeStatus(operation.Status), OperationEventLog.SafeStage(operation.ProgressStage), Math.Clamp(operation.AttemptCount, 0, 10),
Math.Clamp(operation.MaxAttempts, 1, 10), Utc(operation.CreatedAtUtc), Utc(operation.StartedAtUtc), Utc(operation.CompletedAtUtc), Utc(operation.LastHeartbeatAtUtc),
duration, worker, worker != "untracked" && (Enabled("AiOperations") || (worker == "document" && Enabled("CvDocumentImports"))), OperationEventLog.SafeFailure(operation.FailureCategory), job, document);
duration, worker, worker != "untracked" && (Enabled("AiOperations") || (worker == "document" && Enabled("CvDocumentImports"))), OperationEventLog.SafeFailure(operation.FailureCategory), job, document,
Utc(lastFailure?.AtUtc ?? (operation.Status == OperationStatuses.Failed ? operation.CompletedAtUtc : null)),
OperationEventLog.SafeStage(lastFailure?.Stage ?? (operation.Status == OperationStatuses.Failed ? operation.ProgressStage : null)));
}).ToArray();
}
}
JobTrackerApi/Program.cs
+10
View File
@@ -460,6 +460,16 @@ builder.Services.AddRateLimiter(options =>
QueueLimit = 0,
}));
options.AddPolicy(AdminProcessingController.RateLimitPolicy, context =>
RateLimitPartition.GetFixedWindowLimiter(
partitionKey: RateLimitPartitionKeys.Account(context, AdminProcessingController.RateLimitPolicy),
factory: _ => new FixedWindowRateLimiterOptions
{
PermitLimit = 120,
Window = TimeSpan.FromMinutes(1),
QueueLimit = 0,
}));
options.AddPolicy("public-pdf", context =>
RateLimitPartition.GetFixedWindowLimiter(
partitionKey: RateLimitPartitionKeys.PublicPdf(context),
JobTrackerApi/Services/OperationEventLog.cs
+7 -5
View File
@@ -14,11 +14,13 @@ public static class OperationEventLog
public static string? SafeStage(string? value) => value is "cv_validating_file" or "cv_extracting" or "cv_identifying_sections"
or "cv_analysing" or "cv_validating_details" or "cv_preparing_review" or "cv_review_ready" ? value : null;
public static string? SafeFailure(string? value) => value is null ? null : value is
"timeout" or "lease_expired" or "deadline_exceeded" or "entitlement_changed" or "document_unavailable"
or "sidecar_unavailable" or "upstream_unavailable" or "invalid_response" or "circuit_open" or "rate_limited"
or "permission_denied" or "cancelled" or "upload_rejected" or "parser_failed" or "malformed_output" or "provider_unavailable"
? value : "processing_failed";
public static IReadOnlyList<string> FailureCodes { get; } = Array.AsReadOnly(new[]
{
"timeout", "lease_expired", "deadline_exceeded", "entitlement_changed", "document_unavailable",
"sidecar_unavailable", "upstream_unavailable", "invalid_response", "circuit_open", "rate_limited",
"permission_denied", "cancelled", "upload_rejected", "parser_failed", "malformed_output", "provider_unavailable",
});
public static string? SafeFailure(string? value) => value is null ? null : FailureCodes.Contains(value, StringComparer.Ordinal) ? value : "processing_failed";
public static UserOperationEvent Create(UserOperation operation, OperationEventKind kind, DateTime now,
string? status = null, string? stage = null, string? failure = null, int? attempt = null) => new()