fix(admin): complete processing diagnostics and failure evidence

JobTrackerApi/Controllers/AdminProcessingController.cs

@@ -1,10 +1,12 @@
 using System.Globalization;
+using System.Security.Claims;
 using System.Text.Json;
 using JobTrackerApi.Data;
 using JobTrackerApi.Models;
 using JobTrackerApi.Services;
 using Microsoft.AspNetCore.Authorization;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.RateLimiting;
 using Microsoft.EntityFrameworkCore;
 
 namespace JobTrackerApi.Controllers;
@@ -12,15 +14,18 @@ namespace JobTrackerApi.Controllers;
 [ApiController]
 [Route("api/admin/processing")]
 [Authorize(Roles = "Admin")]
+[EnableRateLimiting(RateLimitPolicy)]
 public sealed class AdminProcessingController(JobTrackerContext db, IConfiguration configuration, TimeProvider clock) : ControllerBase
 {
+    public const string RateLimitPolicy = "admin-processing";
     private static readonly HashSet<string> EventKinds = Enum.GetNames<OperationEventKind>().ToHashSet(StringComparer.Ordinal);
     public sealed record IdentityDto(string Id, string? Name, string? Email);
-    public sealed record JobDto(int Id, string Title, string? Company);
-    public sealed record DocumentDto(string Name, string? Type, long Bytes, int? Pages, string? State);
+    public sealed record JobDto(int Id, string Title, string? Company, bool CanOpen);
+    public sealed record DocumentDto(int ImportId, string Name, string? Type, long Bytes, int? Pages, string? State);
     public sealed record ItemDto(Guid Id, IdentityDto Owner, string TaskType, string Status, string? Stage, int AttemptCount,
         int MaxAttempts, DateTime CreatedAtUtc, DateTime? StartedAtUtc, DateTime? CompletedAtUtc, DateTime? LastHeartbeatAtUtc,
-        double DurationSeconds, string Worker, bool WorkerEnabled, string? FailureCode, JobDto? Job, DocumentDto? Document);
+        double DurationSeconds, string Worker, bool WorkerEnabled, string? FailureCode, JobDto? Job, DocumentDto? Document,
+        DateTime? LastFailureAtUtc, string? LastFailureStage);
     public sealed record WorkerDto(string Name, bool FlagEnabled, bool IndividualOperationsRecorded);
     public sealed record PageDto(IReadOnlyList<ItemDto> Items, int Total, int Page, int PageSize,
         IReadOnlyDictionary<string, int> Counts, IReadOnlyList<WorkerDto> Workers, string CvInterpretation);
@@ -29,17 +34,64 @@ public sealed class AdminProcessingController(JobTrackerContext db, IConfigurati
 
     [HttpGet]
     public async Task<ActionResult<PageDto>> List([FromQuery] string? status = null, [FromQuery] string? taskType = null,
-        [FromQuery] string? owner = null, [FromQuery] int page = 1, [FromQuery] int pageSize = 25, CancellationToken cancellationToken = default)
+        [FromQuery] string? owner = null, [FromQuery] int page = 1, [FromQuery] int pageSize = 25,
+        [FromQuery] int? jobId = null, [FromQuery] string? worker = null, [FromQuery] string? failureCode = null,
+        [FromQuery] string? q = null, [FromQuery] DateTimeOffset? createdFrom = null, [FromQuery] DateTimeOffset? createdBefore = null,
+        CancellationToken cancellationToken = default)
     {
         if (page is < 1 or > 1000 || pageSize is < 1 or > 100 || owner?.Length > 255
             || (status is not null && OperationEventLog.SafeStatus(status) == "unknown")
-            || (taskType is not null && taskType is not (CvProcessingQueue.TaskType or StrategySnapshotService.TaskType or "other"))) return BadRequest();
+            || (taskType is not null && taskType is not (CvProcessingQueue.TaskType or StrategySnapshotService.TaskType or "other"))
+            || jobId <= 0 || q?.Length > 200 || (worker is not null && worker is not ("document" or "ai" or "untracked"))
+            || (failureCode is not null && failureCode != "processing_failed" && OperationEventLog.SafeFailure(failureCode) != failureCode)
+            || createdFrom?.UtcDateTime.Year < 1000 || createdBefore?.UtcDateTime.Year < 1000
+            || (createdFrom is not null && createdBefore is not null && createdFrom >= createdBefore)) return BadRequest();
         var all = ReadOperations();
         var query = all;
+        // Historical tables use different MariaDB collations; owner identifiers must compare exactly.
+        var identifiers = db.Database.IsMySql() ? "utf8mb4_bin" : "BINARY";
         if (status is not null) query = query.Where(item => item.Status == status);
         if (!string.IsNullOrWhiteSpace(owner)) query = query.Where(item => item.OwnerUserId == owner);
         if (taskType == "other") query = query.Where(item => item.TaskType != CvProcessingQueue.TaskType && item.TaskType != StrategySnapshotService.TaskType);
         else if (taskType is not null) query = query.Where(item => item.TaskType == taskType);
+        if (createdFrom is not null) { var from = createdFrom.Value.UtcDateTime; query = query.Where(item => item.CreatedAtUtc >= from); }
+        if (createdBefore is not null) { var before = createdBefore.Value.UtcDateTime; query = query.Where(item => item.CreatedAtUtc < before); }
+        if (jobId is not null)
+        {
+            var prefix = jobId.Value.ToString(CultureInfo.InvariantCulture) + "|";
+            query = query.Where(item => item.TaskType == StrategySnapshotService.TaskType && item.SubjectType == "job_strategy"
+                && item.SubjectId != null && item.SubjectId.StartsWith(prefix)
+                && db.JobApplications.IgnoreQueryFilters().Any(job => job.Id == jobId && EF.Functions.Collate(job.OwnerUserId, identifiers) == item.OwnerUserId));
+        }
+        if (worker is not null)
+        {
+            var documents = db.CvExtractionRuns.IgnoreQueryFilters().Where(run => run.ParserVersion == ProfileCvController.DocumentParserVersion && run.Trigger == "document_import");
+            if (worker == "untracked") query = query.Where(item => item.TaskType != CvProcessingQueue.TaskType && item.TaskType != StrategySnapshotService.TaskType);
+            else if (worker == "document") query = query.Where(item => item.TaskType == CvProcessingQueue.TaskType && item.SubjectType == CvProcessingQueue.SubjectType
+                && documents.Any(run => EF.Functions.Collate(run.OwnerUserId, identifiers) == item.OwnerUserId && run.Id.ToString() == item.SubjectId));
+            else query = query.Where(item => (item.TaskType == CvProcessingQueue.TaskType || item.TaskType == StrategySnapshotService.TaskType)
+                && !(item.TaskType == CvProcessingQueue.TaskType && item.SubjectType == CvProcessingQueue.SubjectType
+                    && documents.Any(run => EF.Functions.Collate(run.OwnerUserId, identifiers) == item.OwnerUserId && run.Id.ToString() == item.SubjectId)));
+        }
+        if (failureCode is not null)
+        {
+            var known = OperationEventLog.FailureCodes.ToArray();
+            query = failureCode == "processing_failed"
+                ? query.Where(item => item.FailureCategory != null && !known.Contains(item.FailureCategory))
+                : query.Where(item => item.FailureCategory == failureCode);
+        }
+        if (!string.IsNullOrWhiteSpace(q))
+        {
+            q = q.Trim();
+            var users = db.Users.Where(user => user.Id.Contains(q) || (user.DisplayName != null && user.DisplayName.Contains(q))
+                || (user.Email != null && user.Email.Contains(q))).Select(user => user.Id);
+            var jobs = db.JobApplications.IgnoreQueryFilters().Where(job => job.JobTitle.Contains(q)
+                || (EF.Functions.Collate(job.Company.OwnerUserId, identifiers) == job.OwnerUserId && job.Company.Name.Contains(q)));
+            var isId = Guid.TryParse(q, out var operationId);
+            query = query.Where(item => users.Any(id => EF.Functions.Collate(id, identifiers) == item.OwnerUserId) || (isId && item.Id == operationId)
+                || (item.TaskType == StrategySnapshotService.TaskType && item.SubjectType == "job_strategy" && item.SubjectId != null
+                    && jobs.Any(job => EF.Functions.Collate(job.OwnerUserId, identifiers) == item.OwnerUserId && item.SubjectId.StartsWith(job.Id.ToString() + "|"))));
+        }
         var total = await query.CountAsync(cancellationToken);
         var operations = await query.OrderByDescending(item => item.CreatedAtUtc).ThenByDescending(item => item.Id)
             .Skip((page - 1) * pageSize).Take(pageSize).ToListAsync(cancellationToken);
@@ -127,6 +179,13 @@ public sealed class AdminProcessingController(JobTrackerContext db, IConfigurati
 
     private async Task<IReadOnlyList<ItemDto>> Project(IReadOnlyList<UserOperation> operations, bool includePages, CancellationToken token)
     {
+        var selected = includePages && operations.Count == 1 ? operations[0] : null;
+        var code = OperationEventLog.SafeFailure(selected?.FailureCategory);
+        var cutoff = clock.GetUtcNow().UtcDateTime.AddDays(-OperationEventLog.RetentionDays);
+        var lastFailure = selected is null || code is null ? null : await db.UserOperationEvents.IgnoreQueryFilters().AsNoTracking()
+            .Where(item => item.OperationId == selected.Id && item.OwnerUserId == selected.OwnerUserId && item.AtUtc >= cutoff && item.FailureCode == code
+                && (item.Kind == "Failed" || item.Kind == "RetryScheduled" || item.Kind == "LeaseRecovered" || item.Kind == "Expired"))
+            .OrderByDescending(item => item.Id).Select(item => new { item.AtUtc, item.Stage }).FirstOrDefaultAsync(token);
         var ownerIds = operations.Select(item => item.OwnerUserId).Distinct().ToArray();
         var users = await db.Users.AsNoTracking().Where(item => ownerIds.Contains(item.Id))
             .Select(item => new { item.Id, Name = item.DisplayName ?? item.UserName, item.Email })
@@ -146,9 +205,9 @@ public sealed class AdminProcessingController(JobTrackerContext db, IConfigurati
         {
             var subject = Subject(operation);
             var run = operation.TaskType == CvProcessingQueue.TaskType && subject is int cvId && runs.TryGetValue(cvId, out var foundRun) && foundRun.OwnerUserId == operation.OwnerUserId ? foundRun : null;
-            var job = operation.TaskType == StrategySnapshotService.TaskType && subject is int jobId && jobs.TryGetValue(jobId, out var foundJob) && foundJob.OwnerUserId == operation.OwnerUserId ? new JobDto(foundJob.Id, foundJob.JobTitle, foundJob.Company) : null;
+            var job = operation.TaskType == StrategySnapshotService.TaskType && subject is int jobId && jobs.TryGetValue(jobId, out var foundJob) && foundJob.OwnerUserId == operation.OwnerUserId ? new JobDto(foundJob.Id, foundJob.JobTitle, foundJob.Company, foundJob.OwnerUserId == HttpContext?.User.FindFirstValue(ClaimTypes.NameIdentifier)) : null;
             var artifact = run?.ArtifactId is int artifactId && artifacts.TryGetValue(artifactId, out var foundArtifact) && foundArtifact.OwnerUserId == operation.OwnerUserId ? foundArtifact : null;
-            var document = artifact is null ? null : new DocumentDto(FileName(artifact.OriginalFileName),
+            var document = artifact is null ? null : new DocumentDto(run!.Id, FileName(artifact.OriginalFileName),
                 artifact.MimeType is "application/pdf" or "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ? artifact.MimeType : null,
                 Math.Max(0, artifact.ByteSize), Pages(run?.DocumentJson), run?.Status is "pending_review" or "applied" or "expired" or "failed" or "cancelled" or "running" or "queued" ? run.Status : null);
             var worker = SafeTask(operation.TaskType) == "other" ? "untracked" : run is { ParserVersion: ProfileCvController.DocumentParserVersion, Trigger: "document_import" } ? "document" : "ai";
@@ -156,7 +215,9 @@ public sealed class AdminProcessingController(JobTrackerContext db, IConfigurati
             return new ItemDto(operation.Id, users.GetValueOrDefault(operation.OwnerUserId) ?? new(operation.OwnerUserId, null, null), SafeTask(operation.TaskType),
                 OperationEventLog.SafeStatus(operation.Status), OperationEventLog.SafeStage(operation.ProgressStage), Math.Clamp(operation.AttemptCount, 0, 10),
                 Math.Clamp(operation.MaxAttempts, 1, 10), Utc(operation.CreatedAtUtc), Utc(operation.StartedAtUtc), Utc(operation.CompletedAtUtc), Utc(operation.LastHeartbeatAtUtc),
-                duration, worker, worker != "untracked" && (Enabled("AiOperations") || (worker == "document" && Enabled("CvDocumentImports"))), OperationEventLog.SafeFailure(operation.FailureCategory), job, document);
+                duration, worker, worker != "untracked" && (Enabled("AiOperations") || (worker == "document" && Enabled("CvDocumentImports"))), OperationEventLog.SafeFailure(operation.FailureCategory), job, document,
+                Utc(lastFailure?.AtUtc ?? (operation.Status == OperationStatuses.Failed ? operation.CompletedAtUtc : null)),
+                OperationEventLog.SafeStage(lastFailure?.Stage ?? (operation.Status == OperationStatuses.Failed ? operation.ProgressStage : null)));
         }).ToArray();
     }
 }