cesnimda
224 lines
19 KiB
C#
224 lines
19 KiB
C#
using System.Globalization;
|
|
using System.Security.Claims;
|
|
using System.Text.Json;
|
|
using JobTrackerApi.Data;
|
|
using JobTrackerApi.Models;
|
|
using JobTrackerApi.Services;
|
|
using Microsoft.AspNetCore.Authorization;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Microsoft.AspNetCore.RateLimiting;
|
|
using Microsoft.EntityFrameworkCore;
|
|
|
|
namespace JobTrackerApi.Controllers;
|
|
|
|
[ApiController]
|
|
[Route("api/admin/processing")]
|
|
[Authorize(Roles = "Admin")]
|
|
[EnableRateLimiting(RateLimitPolicy)]
|
|
public sealed class AdminProcessingController(JobTrackerContext db, IConfiguration configuration, TimeProvider clock) : ControllerBase
|
|
{
|
|
public const string RateLimitPolicy = "admin-processing";
|
|
private static readonly HashSet<string> EventKinds = Enum.GetNames<OperationEventKind>().ToHashSet(StringComparer.Ordinal);
|
|
public sealed record IdentityDto(string Id, string? Name, string? Email);
|
|
public sealed record JobDto(int Id, string Title, string? Company, bool CanOpen);
|
|
public sealed record DocumentDto(int ImportId, string Name, string? Type, long Bytes, int? Pages, string? State);
|
|
public sealed record ItemDto(Guid Id, IdentityDto Owner, string TaskType, string Status, string? Stage, int AttemptCount,
|
|
int MaxAttempts, DateTime CreatedAtUtc, DateTime? StartedAtUtc, DateTime? CompletedAtUtc, DateTime? LastHeartbeatAtUtc,
|
|
double DurationSeconds, string Worker, bool WorkerEnabled, string? FailureCode, JobDto? Job, DocumentDto? Document,
|
|
DateTime? LastFailureAtUtc, string? LastFailureStage);
|
|
public sealed record WorkerDto(string Name, bool FlagEnabled, bool IndividualOperationsRecorded);
|
|
public sealed record PageDto(IReadOnlyList<ItemDto> Items, int Total, int Page, int PageSize,
|
|
IReadOnlyDictionary<string, int> Counts, IReadOnlyList<WorkerDto> Workers, string CvInterpretation);
|
|
public sealed record EventDto(string Id, DateTime AtUtc, string Kind, string Status, string? Stage, int Attempt, string? FailureCode);
|
|
public sealed record EventPageDto(IReadOnlyList<EventDto> Items, bool HasMore, int RetentionDays);
|
|
|
|
[HttpGet]
|
|
public async Task<ActionResult<PageDto>> List([FromQuery] string? status = null, [FromQuery] string? taskType = null,
|
|
[FromQuery] string? owner = null, [FromQuery] int page = 1, [FromQuery] int pageSize = 25,
|
|
[FromQuery] int? jobId = null, [FromQuery] string? worker = null, [FromQuery] string? failureCode = null,
|
|
[FromQuery] string? q = null, [FromQuery] DateTimeOffset? createdFrom = null, [FromQuery] DateTimeOffset? createdBefore = null,
|
|
CancellationToken cancellationToken = default)
|
|
{
|
|
if (page is < 1 or > 1000 || pageSize is < 1 or > 100 || owner?.Length > 255
|
|
|| (status is not null && OperationEventLog.SafeStatus(status) == "unknown")
|
|
|| (taskType is not null && taskType is not (CvProcessingQueue.TaskType or StrategySnapshotService.TaskType or "other"))
|
|
|| jobId <= 0 || q?.Length > 200 || (worker is not null && worker is not ("document" or "ai" or "untracked"))
|
|
|| (failureCode is not null && failureCode != "processing_failed" && OperationEventLog.SafeFailure(failureCode) != failureCode)
|
|
|| createdFrom?.UtcDateTime.Year < 1000 || createdBefore?.UtcDateTime.Year < 1000
|
|
|| (createdFrom is not null && createdBefore is not null && createdFrom >= createdBefore)) return BadRequest();
|
|
var all = ReadOperations();
|
|
var query = all;
|
|
// Historical tables use different MariaDB collations; owner identifiers must compare exactly.
|
|
var identifiers = db.Database.IsMySql() ? "utf8mb4_bin" : "BINARY";
|
|
if (status is not null) query = query.Where(item => item.Status == status);
|
|
if (!string.IsNullOrWhiteSpace(owner)) query = query.Where(item => item.OwnerUserId == owner);
|
|
if (taskType == "other") query = query.Where(item => item.TaskType != CvProcessingQueue.TaskType && item.TaskType != StrategySnapshotService.TaskType);
|
|
else if (taskType is not null) query = query.Where(item => item.TaskType == taskType);
|
|
if (createdFrom is not null) { var from = createdFrom.Value.UtcDateTime; query = query.Where(item => item.CreatedAtUtc >= from); }
|
|
if (createdBefore is not null) { var before = createdBefore.Value.UtcDateTime; query = query.Where(item => item.CreatedAtUtc < before); }
|
|
if (jobId is not null)
|
|
{
|
|
var prefix = jobId.Value.ToString(CultureInfo.InvariantCulture) + "|";
|
|
query = query.Where(item => item.TaskType == StrategySnapshotService.TaskType && item.SubjectType == "job_strategy"
|
|
&& item.SubjectId != null && item.SubjectId.StartsWith(prefix)
|
|
&& db.JobApplications.IgnoreQueryFilters().Any(job => job.Id == jobId && EF.Functions.Collate(job.OwnerUserId, identifiers) == item.OwnerUserId));
|
|
}
|
|
if (worker is not null)
|
|
{
|
|
var documents = db.CvExtractionRuns.IgnoreQueryFilters().Where(run => run.ParserVersion == ProfileCvController.DocumentParserVersion && run.Trigger == "document_import");
|
|
if (worker == "untracked") query = query.Where(item => item.TaskType != CvProcessingQueue.TaskType && item.TaskType != StrategySnapshotService.TaskType);
|
|
else if (worker == "document") query = query.Where(item => item.TaskType == CvProcessingQueue.TaskType && item.SubjectType == CvProcessingQueue.SubjectType
|
|
&& documents.Any(run => EF.Functions.Collate(run.OwnerUserId, identifiers) == item.OwnerUserId && run.Id.ToString() == item.SubjectId));
|
|
else query = query.Where(item => (item.TaskType == CvProcessingQueue.TaskType || item.TaskType == StrategySnapshotService.TaskType)
|
|
&& !(item.TaskType == CvProcessingQueue.TaskType && item.SubjectType == CvProcessingQueue.SubjectType
|
|
&& documents.Any(run => EF.Functions.Collate(run.OwnerUserId, identifiers) == item.OwnerUserId && run.Id.ToString() == item.SubjectId)));
|
|
}
|
|
if (failureCode is not null)
|
|
{
|
|
var known = OperationEventLog.FailureCodes.ToArray();
|
|
query = failureCode == "processing_failed"
|
|
? query.Where(item => item.FailureCategory != null && !known.Contains(item.FailureCategory))
|
|
: query.Where(item => item.FailureCategory == failureCode);
|
|
}
|
|
if (!string.IsNullOrWhiteSpace(q))
|
|
{
|
|
q = q.Trim();
|
|
var users = db.Users.Where(user => user.Id.Contains(q) || (user.DisplayName != null && user.DisplayName.Contains(q))
|
|
|| (user.Email != null && user.Email.Contains(q))).Select(user => user.Id);
|
|
var jobs = db.JobApplications.IgnoreQueryFilters().Where(job => job.JobTitle.Contains(q)
|
|
|| (EF.Functions.Collate(job.Company.OwnerUserId, identifiers) == job.OwnerUserId && job.Company.Name.Contains(q)));
|
|
var isId = Guid.TryParse(q, out var operationId);
|
|
query = query.Where(item => users.Any(id => EF.Functions.Collate(id, identifiers) == item.OwnerUserId) || (isId && item.Id == operationId)
|
|
|| (item.TaskType == StrategySnapshotService.TaskType && item.SubjectType == "job_strategy" && item.SubjectId != null
|
|
&& jobs.Any(job => EF.Functions.Collate(job.OwnerUserId, identifiers) == item.OwnerUserId && item.SubjectId.StartsWith(job.Id.ToString() + "|"))));
|
|
}
|
|
var total = await query.CountAsync(cancellationToken);
|
|
var operations = await query.OrderByDescending(item => item.CreatedAtUtc).ThenByDescending(item => item.Id)
|
|
.Skip((page - 1) * pageSize).Take(pageSize).ToListAsync(cancellationToken);
|
|
var grouped = await all.GroupBy(item => item.Status).Select(group => new { Status = group.Key, Count = group.Count() }).ToListAsync(cancellationToken);
|
|
var counts = grouped.GroupBy(item => OperationEventLog.SafeStatus(item.Status)).ToDictionary(group => group.Key, group => group.Sum(item => item.Count));
|
|
var workers = new[]
|
|
{
|
|
new WorkerDto("AiOperations", Enabled("AiOperations"), true),
|
|
new WorkerDto("CvDocumentImports", Enabled("CvDocumentImports"), true),
|
|
new WorkerDto("JobEnrichment", Enabled("JobEnrichment"), false),
|
|
new WorkerDto("Rules", Enabled("Rules"), false),
|
|
new WorkerDto("FollowUpReminders", Enabled("FollowUpReminders"), false),
|
|
new WorkerDto("DailyExport", Enabled("DailyExport"), false),
|
|
new WorkerDto("SavedJobSearchAlerts", configuration.GetValue("Workers:SavedJobSearchAlertsEnabled", true), false),
|
|
};
|
|
return Ok(new PageDto(await Project(operations, false, cancellationToken), total, page, pageSize, counts, workers, "deterministic"));
|
|
}
|
|
|
|
[HttpGet("{id:guid}")]
|
|
public async Task<ActionResult<ItemDto>> Detail(Guid id, CancellationToken cancellationToken)
|
|
{
|
|
var operation = await ReadOperations().SingleOrDefaultAsync(item => item.Id == id, cancellationToken);
|
|
if (operation is null) return NotFound();
|
|
return Ok((await Project(new[] { operation }, true, cancellationToken)).Single());
|
|
}
|
|
|
|
[HttpGet("{id:guid}/events")]
|
|
public async Task<ActionResult<EventPageDto>> Events(Guid id, [FromQuery] long? after = null, [FromQuery] long? before = null,
|
|
[FromQuery] int take = 100, CancellationToken cancellationToken = default)
|
|
{
|
|
if (take is < 1 or > 100 || after < 0 || before < 0 || (after is not null && before is not null)) return BadRequest();
|
|
var owner = await db.UserOperations.IgnoreQueryFilters().Where(item => item.Id == id).Select(item => item.OwnerUserId).SingleOrDefaultAsync(cancellationToken);
|
|
if (owner is null) return NotFound();
|
|
var cutoff = clock.GetUtcNow().UtcDateTime.AddDays(-OperationEventLog.RetentionDays);
|
|
var query = db.UserOperationEvents.IgnoreQueryFilters().AsNoTracking().Where(item => item.OperationId == id && item.OwnerUserId == owner && item.AtUtc >= cutoff);
|
|
if (after is not null) query = query.Where(item => item.Id > after);
|
|
if (before is not null) query = query.Where(item => item.Id < before);
|
|
var rows = await (after is not null ? query.OrderBy(item => item.Id) : query.OrderByDescending(item => item.Id)).Take(take + 1).ToListAsync(cancellationToken);
|
|
var hasMore = rows.Count > take;
|
|
rows = rows.Take(take).ToList();
|
|
if (after is null) rows.Reverse();
|
|
var items = rows.Select(item => new EventDto(item.Id.ToString(CultureInfo.InvariantCulture), Utc(item.AtUtc),
|
|
EventKinds.Contains(item.Kind) ? item.Kind : "Unknown",
|
|
OperationEventLog.SafeStatus(item.Status), OperationEventLog.SafeStage(item.Stage), Math.Clamp(item.Attempt, 0, 10), OperationEventLog.SafeFailure(item.FailureCode))).ToArray();
|
|
return Ok(new EventPageDto(items, hasMore, OperationEventLog.RetentionDays));
|
|
}
|
|
|
|
private bool Enabled(string worker) => configuration.GetValue($"Workers:{worker}Enabled", false);
|
|
private IQueryable<UserOperation> ReadOperations() => db.UserOperations.IgnoreQueryFilters().AsNoTracking().Select(item => new UserOperation
|
|
{
|
|
Id = item.Id, OwnerUserId = item.OwnerUserId, TaskType = item.TaskType, SubjectType = item.SubjectType, SubjectId = item.SubjectId,
|
|
Status = item.Status, ProgressStage = item.ProgressStage, AttemptCount = item.AttemptCount, MaxAttempts = item.MaxAttempts,
|
|
CreatedAtUtc = item.CreatedAtUtc, StartedAtUtc = item.StartedAtUtc, CompletedAtUtc = item.CompletedAtUtc,
|
|
LastHeartbeatAtUtc = item.LastHeartbeatAtUtc, FailureCategory = item.FailureCategory,
|
|
});
|
|
private static DateTime Utc(DateTime value) => DateTime.SpecifyKind(value, DateTimeKind.Utc);
|
|
private static DateTime? Utc(DateTime? value) => value is null ? null : Utc(value.Value);
|
|
private static string SafeTask(string value) => value is CvProcessingQueue.TaskType or StrategySnapshotService.TaskType ? value : "other";
|
|
private static int? Subject(UserOperation operation)
|
|
{
|
|
if (operation.SubjectId?.Length > 128) return null;
|
|
if (operation.TaskType == CvProcessingQueue.TaskType && operation.SubjectType == CvProcessingQueue.SubjectType)
|
|
return int.TryParse(operation.SubjectId, NumberStyles.None, CultureInfo.InvariantCulture, out var id) && id > 0 ? id : null;
|
|
if (operation.TaskType == StrategySnapshotService.TaskType && operation.SubjectType == "job_strategy")
|
|
{
|
|
try { return StrategySnapshotService.DecodeSubject(operation.SubjectId).JobId; }
|
|
catch (AiOperationFailure) { return null; }
|
|
}
|
|
return null;
|
|
}
|
|
private static string FileName(string value) => new string(value.Split('/', '\\').Last().Where(character => !char.IsControl(character)).Take(120).ToArray());
|
|
private static int? Pages(string? json)
|
|
{
|
|
if (json is null || json.Length > 2 * 1024 * 1024) return null;
|
|
try
|
|
{
|
|
using var document = JsonDocument.Parse(json);
|
|
if (document.RootElement.ValueKind == JsonValueKind.Object &&
|
|
(document.RootElement.TryGetProperty("PageCount", out var field) || document.RootElement.TryGetProperty("pageCount", out field)) &&
|
|
field.ValueKind == JsonValueKind.Number && field.TryGetInt32(out var count) && count is > 0 and <= 5000) return count;
|
|
}
|
|
catch (JsonException) { }
|
|
return null;
|
|
}
|
|
|
|
private async Task<IReadOnlyList<ItemDto>> Project(IReadOnlyList<UserOperation> operations, bool includePages, CancellationToken token)
|
|
{
|
|
var selected = includePages && operations.Count == 1 ? operations[0] : null;
|
|
var code = OperationEventLog.SafeFailure(selected?.FailureCategory);
|
|
var cutoff = clock.GetUtcNow().UtcDateTime.AddDays(-OperationEventLog.RetentionDays);
|
|
var lastFailure = selected is null || code is null ? null : await db.UserOperationEvents.IgnoreQueryFilters().AsNoTracking()
|
|
.Where(item => item.OperationId == selected.Id && item.OwnerUserId == selected.OwnerUserId && item.AtUtc >= cutoff && item.FailureCode == code
|
|
&& (item.Kind == "Failed" || item.Kind == "RetryScheduled" || item.Kind == "LeaseRecovered" || item.Kind == "Expired"))
|
|
.OrderByDescending(item => item.Id).Select(item => new { item.AtUtc, item.Stage }).FirstOrDefaultAsync(token);
|
|
var ownerIds = operations.Select(item => item.OwnerUserId).Distinct().ToArray();
|
|
var users = await db.Users.AsNoTracking().Where(item => ownerIds.Contains(item.Id))
|
|
.Select(item => new { item.Id, Name = item.DisplayName ?? item.UserName, item.Email })
|
|
.Select(item => new IdentityDto(item.Id, item.Name != null && item.Name.Length > 160 ? item.Name.Substring(0, 160) + "…" : item.Name,
|
|
item.Email != null && item.Email.Length > 254 ? item.Email.Substring(0, 254) + "…" : item.Email)).ToDictionaryAsync(item => item.Id, token);
|
|
var cvIds = operations.Where(item => item.TaskType == CvProcessingQueue.TaskType).Select(Subject).OfType<int>().ToArray();
|
|
var jobIds = operations.Where(item => item.TaskType == StrategySnapshotService.TaskType).Select(Subject).OfType<int>().ToArray();
|
|
var runs = await db.CvExtractionRuns.IgnoreQueryFilters().AsNoTracking().Where(item => cvIds.Contains(item.Id))
|
|
.Select(item => new { item.Id, item.OwnerUserId, item.ArtifactId, item.ParserVersion, item.Trigger, item.Status, DocumentJson = includePages ? item.DocumentJson : null }).ToDictionaryAsync(item => item.Id, token);
|
|
var artifactIds = runs.Values.Select(item => item.ArtifactId).OfType<int>().ToArray();
|
|
var artifacts = await db.CvUploadArtifacts.IgnoreQueryFilters().AsNoTracking().Where(item => artifactIds.Contains(item.Id))
|
|
.Select(item => new { item.Id, item.OwnerUserId, item.OriginalFileName, item.MimeType, item.ByteSize }).ToDictionaryAsync(item => item.Id, token);
|
|
var jobs = await db.JobApplications.IgnoreQueryFilters().AsNoTracking().Where(item => jobIds.Contains(item.Id))
|
|
.Select(item => new { item.Id, item.OwnerUserId, JobTitle = item.JobTitle.Length > 200 ? item.JobTitle.Substring(0, 200) + "…" : item.JobTitle,
|
|
Company = item.Company.OwnerUserId == item.OwnerUserId ? (item.Company.Name.Length > 160 ? item.Company.Name.Substring(0, 160) + "…" : item.Company.Name) : null }).ToDictionaryAsync(item => item.Id, token);
|
|
return operations.Select(operation =>
|
|
{
|
|
var subject = Subject(operation);
|
|
var run = operation.TaskType == CvProcessingQueue.TaskType && subject is int cvId && runs.TryGetValue(cvId, out var foundRun) && foundRun.OwnerUserId == operation.OwnerUserId ? foundRun : null;
|
|
var job = operation.TaskType == StrategySnapshotService.TaskType && subject is int jobId && jobs.TryGetValue(jobId, out var foundJob) && foundJob.OwnerUserId == operation.OwnerUserId ? new JobDto(foundJob.Id, foundJob.JobTitle, foundJob.Company, foundJob.OwnerUserId == HttpContext?.User.FindFirstValue(ClaimTypes.NameIdentifier)) : null;
|
|
var artifact = run?.ArtifactId is int artifactId && artifacts.TryGetValue(artifactId, out var foundArtifact) && foundArtifact.OwnerUserId == operation.OwnerUserId ? foundArtifact : null;
|
|
var document = artifact is null ? null : new DocumentDto(run!.Id, FileName(artifact.OriginalFileName),
|
|
artifact.MimeType is "application/pdf" or "application/vnd.openxmlformats-officedocument.wordprocessingml.document" ? artifact.MimeType : null,
|
|
Math.Max(0, artifact.ByteSize), Pages(run?.DocumentJson), run?.Status is "pending_review" or "applied" or "expired" or "failed" or "cancelled" or "running" or "queued" ? run.Status : null);
|
|
var worker = SafeTask(operation.TaskType) == "other" ? "untracked" : run is { ParserVersion: ProfileCvController.DocumentParserVersion, Trigger: "document_import" } ? "document" : "ai";
|
|
var duration = operation.StartedAtUtc is DateTime started ? Math.Max(0, ((operation.CompletedAtUtc ?? clock.GetUtcNow().UtcDateTime) - started).TotalSeconds) : 0;
|
|
return new ItemDto(operation.Id, users.GetValueOrDefault(operation.OwnerUserId) ?? new(operation.OwnerUserId, null, null), SafeTask(operation.TaskType),
|
|
OperationEventLog.SafeStatus(operation.Status), OperationEventLog.SafeStage(operation.ProgressStage), Math.Clamp(operation.AttemptCount, 0, 10),
|
|
Math.Clamp(operation.MaxAttempts, 1, 10), Utc(operation.CreatedAtUtc), Utc(operation.StartedAtUtc), Utc(operation.CompletedAtUtc), Utc(operation.LastHeartbeatAtUtc),
|
|
duration, worker, worker != "untracked" && (Enabled("AiOperations") || (worker == "document" && Enabled("CvDocumentImports"))), OperationEventLog.SafeFailure(operation.FailureCategory), job, document,
|
|
Utc(lastFailure?.AtUtc ?? (operation.Status == OperationStatuses.Failed ? operation.CompletedAtUtc : null)),
|
|
OperationEventLog.SafeStage(lastFailure?.Stage ?? (operation.Status == OperationStatuses.Failed ? operation.ProgressStage : null)));
|
|
}).ToArray();
|
|
}
|
|
}
|