feat(cv-import): ground reviewed facts in revisioned career profiles
This commit is contained in:
@@ -0,0 +1,206 @@
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using JobTrackerApi.Models;
|
||||
using JobTrackerApi.Services;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace JobTrackerApi.Controllers;
|
||||
|
||||
public sealed partial class ProfileCvController
|
||||
{
|
||||
public const string DocumentParserVersion = "docling-v1";
|
||||
public sealed record ApplyReviewedRequest(int? ExpectedVersion, IReadOnlyList<CvImportDecision>? Decisions);
|
||||
private sealed record AppliedReview(string Hash, int Revision, StructuredCvProfile StructuredCv, IReadOnlyList<CvImportDecision>? Decisions = null);
|
||||
|
||||
private async Task ProcessDocumentImportAsync(CvExtractionRun run, AiOperationExecutionContext? context, CancellationToken cancellationToken)
|
||||
{
|
||||
async Task Stage(string stage)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
if (context is null) return;
|
||||
var lease = context.Lease;
|
||||
var updated = await _db.UserOperations.Where(o => o.Id == lease.OperationId && o.LeaseToken == lease.LeaseToken && o.Status == OperationStatuses.Running && o.CancellationRequestedAtUtc == null && o.LeaseExpiresAtUtc > DateTime.UtcNow)
|
||||
.ExecuteUpdateAsync(s => s.SetProperty(o => o.ProgressStage, stage), cancellationToken);
|
||||
if (updated != 1) throw new OperationCanceledException("The document processing lease is no longer active.", cancellationToken);
|
||||
}
|
||||
await Stage("cv_extracting");
|
||||
var artifact = await _db.CvUploadArtifacts.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(a => a.Id == run.ArtifactId && a.OwnerUserId == run.OwnerUserId, cancellationToken);
|
||||
if (artifact is null || !System.IO.File.Exists(artifact.StoragePath)) throw new InvalidOperationException("The stored source document is unavailable.");
|
||||
await using var stream = System.IO.File.OpenRead(artifact.StoragePath);
|
||||
var document = await _aiService.ExtractCvDocumentAsync(stream, artifact.OriginalFileName, artifact.MimeType, cancellationToken)
|
||||
?? throw new InvalidOperationException("Document extraction is unavailable. The source was retained for retry.");
|
||||
await Stage("cv_interpreting");
|
||||
var profile = ParseImportDocument(document);
|
||||
await Stage("cv_review_ready");
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var documentJson = JsonSerializer.Serialize(document);
|
||||
var profileJson = StructuredCvProfileJson.Serialize(profile);
|
||||
if (context is null)
|
||||
{
|
||||
run.DocumentJson = documentJson;
|
||||
await CompleteQueuedRunForReviewAsync(run, document.Text, document.Text, profile, cancellationToken);
|
||||
return;
|
||||
}
|
||||
var currentLease = context.Lease;
|
||||
var published = await _db.CvExtractionRuns.Where(r => r.Id == run.Id && r.OwnerUserId == run.OwnerUserId && r.Status == "running" && _db.UserOperations.Any(o => o.Id == currentLease.OperationId && o.LeaseToken == currentLease.LeaseToken && o.Status == OperationStatuses.Running && o.CancellationRequestedAtUtc == null && o.LeaseExpiresAtUtc > DateTime.UtcNow))
|
||||
.ExecuteUpdateAsync(s => s.SetProperty(r => r.DocumentJson, documentJson).SetProperty(r => r.RawExtractedText, document.Text).SetProperty(r => r.NormalizedText, document.Text).SetProperty(r => r.StructuredProfileJson, profileJson).SetProperty(r => r.Status, "pending_review").SetProperty(r => r.CompletedAtUtc, DateTimeOffset.UtcNow), cancellationToken);
|
||||
if (published != 1) throw new OperationCanceledException("The document processing lease was lost before publication.", cancellationToken);
|
||||
await _db.Entry(run).ReloadAsync(cancellationToken);
|
||||
await PruneExtractionRunsAsync(run.OwnerUserId, cancellationToken);
|
||||
}
|
||||
|
||||
[HttpPost("import")]
|
||||
[Authorize(Policy = ProEntitlement.Policy)]
|
||||
[RequestSizeLimit(MaxFileSizeBytes + 65536)]
|
||||
public async Task<IActionResult> Import([FromForm] IFormFile file)
|
||||
{
|
||||
var user = await _users.GetUserAsync(User);
|
||||
if (user is null) return Unauthorized();
|
||||
if (file is null || file.Length is <= 0 or > MaxFileSizeBytes) return BadRequest("Select a PDF or DOCX CV under 5 MB.");
|
||||
var extension = Path.GetExtension(file.FileName).ToLowerInvariant();
|
||||
if (extension is not (".pdf" or ".docx")) return BadRequest("Select a PDF or DOCX CV.");
|
||||
try { await ValidateImportAsync(file, extension, HttpContext.RequestAborted); }
|
||||
catch (InvalidDataException) { return BadRequest("The document signature or archive is invalid or unsupported."); }
|
||||
if (await _db.CvExtractionRuns.CountAsync(r => r.OwnerUserId == user.Id && r.Status == "pending_review", HttpContext.RequestAborted) >= ExtractionRunRetentionCount) return Conflict("Review or discard an existing import before uploading another document.");
|
||||
var artifact = await SaveUploadArtifactAsync(user, file, HttpContext.RequestAborted);
|
||||
artifact.OriginalFileName = Path.GetFileName(file.FileName.Replace('\\', '/'));
|
||||
artifact.MimeType = extension == ".pdf" ? "application/pdf" : "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
||||
try
|
||||
{
|
||||
var active = await FindActiveRunAsync(user.Id, "document_import", null, artifact.Sha256, HttpContext.RequestAborted);
|
||||
if (active is not null) { TryDeleteCvArtifactFile(artifact.StoragePath); return await EnqueueRunAsync(active, HttpContext.RequestAborted); }
|
||||
var run = new CvExtractionRun { OwnerUserId = user.Id, Artifact = artifact, Trigger = "document_import", ParserVersion = DocumentParserVersion, NormalizerVersion = "source-grounded-v1", LlmPromptVersion = "none", Status = "queued" };
|
||||
_db.CvExtractionRuns.Add(run);
|
||||
await _db.SaveChangesAsync(HttpContext.RequestAborted);
|
||||
return await EnqueueRunAsync(run, HttpContext.RequestAborted);
|
||||
}
|
||||
catch { if (artifact.Id == 0) TryDeleteCvArtifactFile(artifact.StoragePath); throw; }
|
||||
}
|
||||
|
||||
private static async Task ValidateImportAsync(IFormFile file, string extension, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var source = file.OpenReadStream();
|
||||
using var bytes = new MemoryStream();
|
||||
var buffer = new byte[8192];
|
||||
int read;
|
||||
while ((read = await source.ReadAsync(buffer, cancellationToken)) > 0)
|
||||
{
|
||||
if (bytes.Length + read > MaxFileSizeBytes) throw new InvalidDataException();
|
||||
bytes.Write(buffer, 0, read);
|
||||
}
|
||||
var data = bytes.ToArray();
|
||||
if (extension == ".pdf")
|
||||
{
|
||||
if (data.Length < 8 || !data.AsSpan(0, 5).SequenceEqual("%PDF-"u8)) throw new InvalidDataException();
|
||||
var tokens = Encoding.Latin1.GetString(data);
|
||||
if (new[] { "/JavaScript", "/JS", "/Launch", "/OpenAction", "/EmbeddedFile", "/Encrypt" }.Any(tokens.Contains)) throw new InvalidDataException();
|
||||
return;
|
||||
}
|
||||
bytes.Position = 0;
|
||||
try
|
||||
{
|
||||
using var archive = new ZipArchive(bytes, ZipArchiveMode.Read, true);
|
||||
if (archive.Entries.Count is 0 or > 2048 || archive.GetEntry("word/document.xml") is null || archive.GetEntry("[Content_Types].xml") is null) throw new InvalidDataException();
|
||||
long expanded = 0;
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
expanded += entry.Length;
|
||||
if (entry.FullName.Contains("..") || entry.FullName.StartsWith('/') || entry.FullName.Contains('\\') || entry.Length > 10 * 1024 * 1024 || expanded > 25 * 1024 * 1024 || entry.Length > 200 * Math.Max(1, entry.CompressedLength) || entry.FullName.EndsWith("vbaProject.bin", StringComparison.OrdinalIgnoreCase)) throw new InvalidDataException();
|
||||
if (entry.FullName.EndsWith(".rels", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
using var reader = new StreamReader(entry.Open());
|
||||
var relationships = await reader.ReadToEndAsync(cancellationToken);
|
||||
// Hyperlinks remain inert text; linked templates/OLE/media are refused.
|
||||
var xml = System.Xml.Linq.XDocument.Parse(relationships);
|
||||
if (xml.Descendants().Any(e => string.Equals((string?)e.Attribute("TargetMode"), "External", StringComparison.OrdinalIgnoreCase) && !((string?)e.Attribute("Type") ?? "").EndsWith("/hyperlink", StringComparison.Ordinal))) throw new InvalidDataException();
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or System.Xml.XmlException or NotSupportedException) { throw new InvalidDataException("Invalid document archive.", ex); }
|
||||
}
|
||||
|
||||
[NonAction]
|
||||
public static StructuredCvProfile ParseImportDocument(CvImportDocument document)
|
||||
{
|
||||
var candidate = CvDocumentInterpreter.Parse(document.Blocks);
|
||||
return CvDocumentReview.Ground(candidate, document);
|
||||
}
|
||||
|
||||
[HttpGet("runs/{id:int}/review")]
|
||||
public async Task<IActionResult> GetImportReview(int id)
|
||||
{
|
||||
var user = await _users.GetUserAsync(User);
|
||||
if (user is null) return Unauthorized();
|
||||
var run = await _db.CvExtractionRuns.AsNoTracking().Include(r => r.Artifact).FirstOrDefaultAsync(r => r.Id == id && r.OwnerUserId == user.Id, HttpContext.RequestAborted);
|
||||
if (run is null) return NotFound();
|
||||
if (run.ParserVersion != DocumentParserVersion || string.IsNullOrWhiteSpace(run.StructuredProfileJson)) return Conflict("This import is not ready for review.");
|
||||
var snapshot = await _careerProfileService.LoadSnapshotAsync(user.Id, HttpContext.RequestAborted);
|
||||
var document = JsonSerializer.Deserialize<CvImportDocument>(run.DocumentJson ?? "null");
|
||||
return Ok(new CvImportReview(run.Id, run.Status, snapshot.Revision, run.Artifact?.OriginalFileName, $"/api/profile-cv/runs/{run.Id}/source", run.RawExtractedText, document?.Warnings ?? Array.Empty<string>(), CvDocumentReview.Build(snapshot.Profile, StructuredCvProfileJson.Deserialize(run.StructuredProfileJson))));
|
||||
}
|
||||
|
||||
[HttpGet("runs/{id:int}/source")]
|
||||
public async Task<IActionResult> GetImportSource(int id)
|
||||
{
|
||||
var user = await _users.GetUserAsync(User);
|
||||
if (user is null) return Unauthorized();
|
||||
var run = await _db.CvExtractionRuns.AsNoTracking().Include(r => r.Artifact).FirstOrDefaultAsync(r => r.Id == id && r.OwnerUserId == user.Id, HttpContext.RequestAborted);
|
||||
var artifact = run?.Artifact;
|
||||
if (artifact is null || artifact.OwnerUserId != user.Id || !System.IO.File.Exists(artifact.StoragePath)) return NotFound();
|
||||
Response.Headers.CacheControl = "private, no-store";
|
||||
Response.Headers["X-Content-Type-Options"] = "nosniff";
|
||||
return PhysicalFile(artifact.StoragePath, artifact.MimeType, artifact.OriginalFileName);
|
||||
}
|
||||
|
||||
[HttpPost("runs/{id:int}/apply-reviewed")]
|
||||
public async Task<IActionResult> ApplyReviewed(int id, [FromBody] ApplyReviewedRequest? request)
|
||||
{
|
||||
var user = await _users.GetUserAsync(User);
|
||||
if (user is null) return Unauthorized();
|
||||
if (request?.ExpectedVersion is null or < 0 || request.Decisions is null || request.Decisions.Any(d => d is null || d.Id is null || d.Action is null)) return BadRequest("expectedVersion and review decisions are required.");
|
||||
var hash = Convert.ToHexString(SHA256.HashData(JsonSerializer.SerializeToUtf8Bytes(new { request.ExpectedVersion, Decisions = request.Decisions.OrderBy(d => d.Id, StringComparer.Ordinal) })));
|
||||
try
|
||||
{
|
||||
return await _careerProfileService.ExecuteAtomicAsync<IActionResult>(async () =>
|
||||
{
|
||||
var run = await _db.CvExtractionRuns.FirstOrDefaultAsync(r => r.Id == id && r.OwnerUserId == user.Id, HttpContext.RequestAborted);
|
||||
if (run is null) return NotFound();
|
||||
if (run.ParserVersion != DocumentParserVersion) return Conflict("Use legacy review for this run.");
|
||||
if (run.Status == "applied")
|
||||
{
|
||||
var prior = JsonSerializer.Deserialize<AppliedReview>(run.ReviewJson ?? "null");
|
||||
return prior?.Hash == hash ? Ok(new { runId = id, status = "applied", revision = prior.Revision, structuredCv = prior.StructuredCv }) : Conflict("This import has already been applied with different decisions.");
|
||||
}
|
||||
if (run.Status != "pending_review" || string.IsNullOrWhiteSpace(run.StructuredProfileJson)) return Conflict("This import is not awaiting review.");
|
||||
var snapshot = await _careerProfileService.LoadSnapshotAsync(user.Id, HttpContext.RequestAborted);
|
||||
if (snapshot.Revision != request.ExpectedVersion) throw new DbUpdateConcurrencyException();
|
||||
var merged = CvDocumentReview.Apply(snapshot.Profile, StructuredCvProfileJson.Deserialize(run.StructuredProfileJson), request.Decisions);
|
||||
if (!request.Decisions.Any(d => d.Action != "keep")) return BadRequest("No imported fields were selected. Discard this import to keep your current profile.");
|
||||
var validation = CareerProfileValidator.Validate(merged);
|
||||
if (validation is not null || StructuredCvProfileJson.SerializePersisted(merged).Length > 500000) return BadRequest(validation ?? "The reviewed profile exceeds its size limit.");
|
||||
merged.Metadata.ProfileVersion = (user.CurrentCvProfileVersion ?? 0) + 1;
|
||||
merged.Metadata.AppliedExtractionRunId = run.Id;
|
||||
merged.Metadata.UpdatedAtUtc = DateTimeOffset.UtcNow;
|
||||
await _careerProfileService.SaveVersionAsync(user.Id, merged, "document_import:reviewed", request.ExpectedVersion.Value, HttpContext.RequestAborted);
|
||||
user.ProfileCvStructureJson = StructuredCvProfileJson.SerializePersisted(merged);
|
||||
user.CurrentCvExtractionRunId = run.Id;
|
||||
user.CurrentCvProfileVersion = merged.Metadata.ProfileVersion;
|
||||
user.CurrentCvUploadArtifactId = run.ArtifactId;
|
||||
run.Status = "applied";
|
||||
run.AppliedAtUtc = DateTimeOffset.UtcNow;
|
||||
run.ReviewJson = JsonSerializer.Serialize(new AppliedReview(hash, snapshot.Revision + 1, merged, request.Decisions));
|
||||
var update = await _users.UpdateAsync(user);
|
||||
if (!update.Succeeded) throw new CareerProjectionException(string.Join("; ", update.Errors.Select(e => e.Description)));
|
||||
await _db.SaveChangesAsync(HttpContext.RequestAborted);
|
||||
return Ok(new { runId = id, run.Status, revision = snapshot.Revision + 1, structuredCv = merged });
|
||||
}, HttpContext.RequestAborted);
|
||||
}
|
||||
catch (DbUpdateConcurrencyException) { return Conflict(new { code = "revision_conflict", message = "The career profile changed. Refresh the comparison and review your decisions." }); }
|
||||
catch (ArgumentException ex) { return BadRequest(ex.Message); }
|
||||
catch (CareerProjectionException ex) { return BadRequest(ex.Message); }
|
||||
}
|
||||
}
|
||||
@@ -184,11 +184,13 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
var storedFileName = $"{DateTimeOffset.UtcNow:yyyyMMddHHmmss}-{Guid.NewGuid():N}{extension}";
|
||||
var storagePath = Path.Combine(userRoot, storedFileName);
|
||||
|
||||
await using (var target = System.IO.File.Create(storagePath))
|
||||
await using (var source = file.OpenReadStream())
|
||||
try
|
||||
{
|
||||
await using var target = System.IO.File.Create(storagePath);
|
||||
await using var source = file.OpenReadStream();
|
||||
await source.CopyToAsync(target, cancellationToken);
|
||||
}
|
||||
catch { TryDeleteCvArtifactFile(storagePath); throw; }
|
||||
|
||||
await using var hashStream = System.IO.File.OpenRead(storagePath);
|
||||
var shaBytes = await SHA256.HashDataAsync(hashStream, cancellationToken);
|
||||
@@ -402,7 +404,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
// transient service). NonAction keeps it off the HTTP surface: without it the
|
||||
// controller-level [Route] exposes it as an any-verb endpoint.
|
||||
[NonAction]
|
||||
public async Task<CvProcessingOutcome?> ProcessQueuedRunAsync(int runId, CancellationToken cancellationToken)
|
||||
public async Task<CvProcessingOutcome?> ProcessQueuedRunAsync(int runId, CancellationToken cancellationToken, AiOperationExecutionContext? operationContext = null)
|
||||
{
|
||||
var ownerUserId = _db.CurrentUserId;
|
||||
var run = ownerUserId is null
|
||||
@@ -439,6 +441,11 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
AiGenerationResult? generation = null;
|
||||
switch (run.Trigger)
|
||||
{
|
||||
case "document_import":
|
||||
{
|
||||
await ProcessDocumentImportAsync(run, operationContext, cancellationToken);
|
||||
break;
|
||||
}
|
||||
case "rebuild":
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(user.ProfileCvText)) throw new InvalidOperationException("Add or import CV text before rebuilding it.");
|
||||
@@ -556,7 +563,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
private async Task PruneExtractionRunsAsync(string ownerUserId, CancellationToken cancellationToken)
|
||||
{
|
||||
var completedRuns = _db.CvExtractionRuns.IgnoreQueryFilters()
|
||||
.Where(x => x.OwnerUserId == ownerUserId && x.Status != "queued" && x.Status != "running");
|
||||
.Where(x => x.OwnerUserId == ownerUserId && x.Status != "queued" && x.Status != "running" && x.Status != "pending_review");
|
||||
var expired = _db.Database.IsSqlite()
|
||||
? (await completedRuns.ToListAsync(cancellationToken)).OrderByDescending(x => x.StartedAtUtc).Skip(ExtractionRunRetentionCount).ToList()
|
||||
: await completedRuns.OrderByDescending(x => x.StartedAtUtc).Skip(ExtractionRunRetentionCount).ToListAsync(cancellationToken);
|
||||
|
||||
@@ -279,6 +279,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
{
|
||||
var run = await _db.CvExtractionRuns.FirstOrDefaultAsync(x => x.Id == id && x.OwnerUserId == user.Id, HttpContext.RequestAborted);
|
||||
if (run is null) return NotFound();
|
||||
if (run.ParserVersion == DocumentParserVersion) return Conflict("This document import requires explicit field review.");
|
||||
if (run.Status != "pending_review") return Conflict("This extraction run is not awaiting review.");
|
||||
if (string.IsNullOrWhiteSpace(run.StructuredProfileJson)) return Conflict("This extraction run has no reviewable profile.");
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
using JobTrackerApi.Data;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace JobTrackerApi.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
[DbContext(typeof(JobTrackerContext))]
|
||||
[Migration("20261026090000_AddCvImportReviewEvidence")]
|
||||
public sealed class AddCvImportReviewEvidence : Migration
|
||||
{
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
var text = ActiveProvider.Contains("MySql", StringComparison.OrdinalIgnoreCase) ? "longtext" : "TEXT";
|
||||
migrationBuilder.AddColumn<string>("DocumentJson", "CvExtractionRuns", type: text, nullable: true);
|
||||
migrationBuilder.AddColumn<string>("ReviewJson", "CvExtractionRuns", type: text, nullable: true);
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
foreach (var column in new[] { "DocumentJson", "ReviewJson" })
|
||||
if (ActiveProvider.Contains("Sqlite", StringComparison.OrdinalIgnoreCase)) migrationBuilder.Sql($"ALTER TABLE \"CvExtractionRuns\" DROP COLUMN \"{column}\";");
|
||||
else migrationBuilder.DropColumn(column, "CvExtractionRuns");
|
||||
}
|
||||
}
|
||||
@@ -1186,6 +1186,8 @@ namespace JobTrackerApi.Migrations
|
||||
|
||||
modelBuilder.Entity("JobTrackerApi.Models.CvExtractionRun", b =>
|
||||
{
|
||||
b.Property<string>("DocumentJson").HasColumnType("TEXT");
|
||||
b.Property<string>("ReviewJson").HasColumnType("TEXT");
|
||||
b.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
@@ -27,6 +27,8 @@ public sealed class CvExtractionRun
|
||||
public string? RawExtractedText { get; set; }
|
||||
public string? NormalizedText { get; set; }
|
||||
public string? StructuredProfileJson { get; set; }
|
||||
public string? DocumentJson { get; set; }
|
||||
public string? ReviewJson { get; set; }
|
||||
public string? ErrorMessage { get; set; }
|
||||
public DateTimeOffset StartedAtUtc { get; set; } = DateTimeOffset.UtcNow;
|
||||
public DateTimeOffset? CompletedAtUtc { get; set; }
|
||||
|
||||
@@ -166,6 +166,7 @@ builder.Services.AddHostedService<DailyExportHostedService>();
|
||||
builder.Services.AddHostedService<JobEnrichmentHostedService>();
|
||||
builder.Services.AddHostedService<SummarizerProbeHostedService>();
|
||||
builder.Services.AddHostedService<AiOperationHostedService>();
|
||||
builder.Services.AddHostedService<CvImportRetentionHostedService>();
|
||||
builder.Services.AddHostedService<EmailSendAttemptRecoveryHostedService>();
|
||||
builder.Services.AddHostedService<AccountDeletionHostedService>();
|
||||
|
||||
|
||||
@@ -241,9 +241,10 @@ public sealed class AiOperationWorker(
|
||||
execution.Cancel();
|
||||
return;
|
||||
}
|
||||
await store.HeartbeatAsync(lease.OperationId, lease.LeaseToken,
|
||||
var renewed = await store.HeartbeatAsync(lease.OperationId, lease.LeaseToken,
|
||||
TimeSpan.FromSeconds(Math.Clamp(configuration.GetValue("AiQueue:LeaseSeconds", 120), 30, 1800)),
|
||||
"processing", null, cancellationToken);
|
||||
null, null, cancellationToken);
|
||||
if (renewed != 1) { execution.Cancel(); return; }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using JobTrackerApi.Models;
|
||||
|
||||
namespace JobTrackerApi.Services;
|
||||
|
||||
// Structural evidence is interpreted before any profile is proposed; ambiguous source stays reviewable.
|
||||
public static class CvDocumentInterpreter
|
||||
{
|
||||
private static readonly Dictionary<string, string> Headings = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["contact"] = "contact", ["kontakt"] = "contact", ["contact details"] = "contact",
|
||||
["professional summary"] = "summary", ["summary"] = "summary", ["profile"] = "summary", ["profil"] = "summary", ["sammendrag"] = "summary",
|
||||
["work experience"] = "jobs", ["experience"] = "jobs", ["employment history"] = "jobs", ["professional experience"] = "jobs", ["arbeidserfaring"] = "jobs", ["yrkeserfaring"] = "jobs", ["erfaring"] = "jobs",
|
||||
["education"] = "education", ["utdanning"] = "education", ["qualifications"] = "education",
|
||||
["skills"] = "skills", ["technical skills"] = "skills", ["core skills"] = "skills", ["ferdigheter"] = "skills", ["kompetanse"] = "skills", ["tekniske ferdigheter"] = "skills",
|
||||
["languages"] = "languages", ["språk"] = "languages", ["certifications"] = "certifications", ["sertifiseringer"] = "certifications",
|
||||
["projects"] = "projects", ["prosjekter"] = "projects", ["interests"] = "interests", ["interesser"] = "interests",
|
||||
["awards"] = "awards", ["publications"] = "publications", ["organisations"] = "organisations", ["organizations"] = "organisations", ["references"] = "references", ["referanser"] = "references",
|
||||
["courses"] = "Courses", ["kurs"] = "Kurs", ["volunteer experience"] = "Volunteer Experience", ["notes"] = "Notes",
|
||||
};
|
||||
|
||||
public static StructuredCvProfile Parse(IReadOnlyList<CvImportBlock> blocks)
|
||||
{
|
||||
var profile = new StructuredCvProfile();
|
||||
var section = "contact";
|
||||
var sectionTitle = "Contact";
|
||||
var unassigned = new List<string>();
|
||||
void Unknown(string line)
|
||||
{
|
||||
var other = profile.OtherSections.FirstOrDefault(s => s.Title == sectionTitle);
|
||||
if (other is null) { other = new() { Title = sectionTitle }; profile.OtherSections.Add(other); }
|
||||
other.Items.Add(line);
|
||||
}
|
||||
foreach (var block in blocks)
|
||||
foreach (var source in block.Text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var line = source.Trim().TrimStart('#', '•').Trim();
|
||||
if (line.StartsWith('|')) line = line.Trim('|').Trim();
|
||||
if (Regex.IsMatch(line, @"^[|:\s-]+$")) continue;
|
||||
var heading = line.TrimEnd(':');
|
||||
if (Headings.TryGetValue(heading, out var named)) { section = named; sectionTitle = heading; continue; }
|
||||
if (block.Kind == "section_header") { section = "custom"; sectionTitle = heading; continue; }
|
||||
var contact = Regex.Match(line, @"^(name|navn|full name|email|e-post|phone|telefon|location|sted|headline|title|website|nettside|linkedin|github|portfolio)\s*:\s*(.+)$", RegexOptions.IgnoreCase);
|
||||
if (section == "contact" && contact.Success)
|
||||
{
|
||||
var value = contact.Groups[2].Value.Trim();
|
||||
switch (contact.Groups[1].Value.ToLowerInvariant())
|
||||
{
|
||||
case "name": case "navn": case "full name": profile.Contact.FullName = value; break;
|
||||
case "email": case "e-post": profile.Contact.Email = value; break;
|
||||
case "phone": case "telefon": profile.Contact.Phone = value; break;
|
||||
case "location": case "sted": profile.Contact.Location = value; break;
|
||||
case "headline": case "title": profile.Contact.Headline = value; break;
|
||||
case "website": case "nettside": profile.Contact.Website = value; break;
|
||||
case "linkedin": profile.Contact.LinkedIn = value; break;
|
||||
case "github": profile.Contact.GitHub = value; break;
|
||||
case "portfolio": profile.Contact.Links.Add(new() { Label = contact.Groups[1].Value, Url = value }); break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
var parts = line.Split('|', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
switch (section)
|
||||
{
|
||||
case "contact":
|
||||
var email = Regex.Match(line, @"[\w.+-]+@[\w.-]+\.[\p{L}]{2,}");
|
||||
if (email.Success) profile.Contact.Email ??= email.Value;
|
||||
else if (block.Kind == "title" && profile.Contact.FullName is null) profile.Contact.FullName = line;
|
||||
else unassigned.Add(line);
|
||||
break;
|
||||
case "summary": profile.Summary.Add(line); break;
|
||||
case "skills": profile.Skills.AddRange(Regex.Split(line, @"\s*[,;•|]\s*").Where(s => s.Length > 0)); break;
|
||||
case "languages":
|
||||
var language = line.Split(':', 2, StringSplitOptions.TrimEntries);
|
||||
profile.Languages.Add(new() { Name = language[0], Level = language.Length == 2 ? language[1] : null }); break;
|
||||
case "jobs":
|
||||
if (parts.Length >= 2)
|
||||
{
|
||||
var (start, end) = Dates(parts.ElementAtOrDefault(2));
|
||||
profile.Jobs.Add(new() { Title = parts[0], Company = parts[1], Start = start, End = end, IsCurrent = end is not null && Regex.IsMatch(end, @"^(present|current|now|nå|dags dato|pågående)$", RegexOptions.IgnoreCase) });
|
||||
}
|
||||
else if (profile.Jobs.Count > 0) profile.Jobs[^1].Bullets.Add(line.TrimStart('-', ' '));
|
||||
else Unknown(line);
|
||||
break;
|
||||
case "education":
|
||||
if (parts.Length >= 2)
|
||||
{
|
||||
var (start, end) = Dates(parts.ElementAtOrDefault(2));
|
||||
profile.Education.Add(new() { Qualification = parts[0], Institution = parts[1], Start = start, End = end });
|
||||
}
|
||||
else if (profile.Education.Count > 0) profile.Education[^1].Details.Add(line);
|
||||
else Unknown(line);
|
||||
break;
|
||||
case "certifications": profile.Certifications.Add(new() { Name = parts[0], Issuer = parts.ElementAtOrDefault(1), Date = parts.ElementAtOrDefault(2) }); break;
|
||||
case "projects":
|
||||
if (profile.Projects.Count == 0 || parts.Length > 1 || block.Kind == "title") profile.Projects.Add(new() { Name = parts[0], Role = parts.ElementAtOrDefault(1) });
|
||||
else profile.Projects[^1].Bullets.Add(line);
|
||||
break;
|
||||
case "interests": profile.Interests.Add(line); break;
|
||||
case "awards": profile.Awards.Add(line); break;
|
||||
case "publications": profile.Publications.Add(line); break;
|
||||
case "organisations": profile.Organisations.Add(line); break;
|
||||
case "references": profile.References.Add(line); break;
|
||||
default: Unknown(line); break;
|
||||
}
|
||||
}
|
||||
if (unassigned.Count > 0) profile.OtherSections.Add(new() { Title = "Unclassified source", Items = unassigned });
|
||||
return profile;
|
||||
}
|
||||
|
||||
private static (string? Start, string? End) Dates(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text)) return (null, null);
|
||||
var dates = Regex.Split(text, @"\s+(?:-|–|—|to|til)\s+|(?<=\d{4})[–—](?=\d{4})", RegexOptions.IgnoreCase);
|
||||
return dates.Length == 2 ? (dates[0].Trim(), dates[1].Trim()) : (text.Trim(), null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
using JobTrackerApi.Models;
|
||||
|
||||
namespace JobTrackerApi.Services;
|
||||
|
||||
public sealed record CvImportBlock(string Id, string Text, string Kind, int? Page);
|
||||
public sealed record CvImportDocument(string Text, bool OcrUsed, string? ContentType, int? PageCount, int Characters, string? FileName, IReadOnlyList<CvImportBlock> Blocks, IReadOnlyList<string> Warnings, string Method);
|
||||
public sealed record CvImportDecision(string Id, string Action, string? Value = null);
|
||||
public sealed record CvImportField(string Id, string Label, string? CurrentValue, string ExtractedValue, string? SourceSnippet, int? SourcePage, string Method, string ReviewState);
|
||||
public sealed record CvImportGroup(string Id, string Category, string Label, string? MatchDescription, IReadOnlyList<CvImportField> Fields);
|
||||
public sealed record CvImportReview(int RunId, string Status, int Revision, string? FileName, string SourceUrl, string? SourceText, IReadOnlyList<string> Warnings, IReadOnlyList<CvImportGroup> Groups);
|
||||
|
||||
// Paths are generated from the canonical typed schema, never accepted as arbitrary JSON patches.
|
||||
public static class CvDocumentReview
|
||||
{
|
||||
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web);
|
||||
private static readonly HashSet<string> Excluded = new(StringComparer.Ordinal) { "id", "metadata", "version", "sections", "startDate", "endDate", "dateNormalized" };
|
||||
private static JsonObject Node(StructuredCvProfile profile) => JsonSerializer.SerializeToNode(profile, Json)!.AsObject();
|
||||
private static string Clean(string text) => Regex.Replace(text, @"\s+", " ").Trim();
|
||||
|
||||
public static StructuredCvProfile Ground(StructuredCvProfile candidate, CvImportDocument document)
|
||||
{
|
||||
var node = Node(candidate);
|
||||
node["sections"] = new JsonArray();
|
||||
var metadata = new Dictionary<string, StructuredCvFieldMetadata>();
|
||||
void Walk(JsonNode? value, string path)
|
||||
{
|
||||
if (value is JsonObject obj)
|
||||
{
|
||||
foreach (var entry in obj.ToList())
|
||||
{
|
||||
if (Excluded.Contains(entry.Key)) { if (entry.Key is "startDate" or "endDate" or "dateNormalized" or "id") obj[entry.Key] = null; continue; }
|
||||
Walk(entry.Value, path.Length == 0 ? entry.Key : path + "." + entry.Key);
|
||||
}
|
||||
}
|
||||
else if (value is JsonArray array)
|
||||
{
|
||||
for (var i = array.Count - 1; i >= 0; i--)
|
||||
{
|
||||
if (array[i] is JsonValue scalar && scalar.TryGetValue<string>(out var text) && !document.Blocks.Any(b => Clean(b.Text).Contains(Clean(text), StringComparison.OrdinalIgnoreCase))) array.RemoveAt(i);
|
||||
}
|
||||
for (var i = 0; i < array.Count; i++) Walk(array[i], path + "." + i);
|
||||
}
|
||||
else if (value is JsonValue scalar)
|
||||
{
|
||||
if (scalar.TryGetValue<bool>(out var flag))
|
||||
{
|
||||
if (flag && value.Parent is JsonObject record)
|
||||
{
|
||||
var end = record["end"]?.ToString();
|
||||
var marker = end is not null && Regex.IsMatch(end, @"^(present|current|now|nå|dags dato|pågående)$", RegexOptions.IgnoreCase);
|
||||
var evidence = marker ? document.Blocks.FirstOrDefault(b => Clean(b.Text).Contains(Clean(end!), StringComparison.OrdinalIgnoreCase)) : null;
|
||||
if (evidence is null) record[path.Split('.').Last()] = false;
|
||||
else metadata[path] = new() { Method = "normalized-current-marker", SourceBlockId = evidence.Id, SourcePage = evidence.Page, SourceSnippet = end, ReviewState = "suggested" };
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!scalar.TryGetValue<string>(out var text) || string.IsNullOrWhiteSpace(text)) return;
|
||||
var block = document.Blocks.FirstOrDefault(b => Clean(b.Text).Contains(Clean(text), StringComparison.OrdinalIgnoreCase));
|
||||
if (block is null) { if (value.Parent is JsonObject parent) parent[path.Split('.').Last()] = null; return; }
|
||||
metadata[path] = new() { Method = "deterministic-source", SourceBlockId = block.Id, SourcePage = block.Page, SourceSnippet = text.Length > 500 ? text[..500] : text, ReviewState = "suggested" };
|
||||
}
|
||||
}
|
||||
Walk(node, "");
|
||||
var result = node.Deserialize<StructuredCvProfile>(Json)!;
|
||||
result.Metadata = new() { Fields = metadata };
|
||||
return result;
|
||||
}
|
||||
|
||||
private sealed record GroupMap(CvImportGroup Review, JsonNode Incoming, string Category, int? ExistingIndex, bool IsArray);
|
||||
public static IReadOnlyList<CvImportGroup> Build(StructuredCvProfile current, StructuredCvProfile incoming) => Maps(current, incoming).Select(m => m.Review).ToList();
|
||||
|
||||
private static List<GroupMap> Maps(StructuredCvProfile current, StructuredCvProfile incoming)
|
||||
{
|
||||
var old = Node(current);
|
||||
var next = Node(incoming);
|
||||
var groups = new List<GroupMap>();
|
||||
foreach (var category in next)
|
||||
{
|
||||
if (Excluded.Contains(category.Key) || category.Value is null) continue;
|
||||
if (category.Value is JsonArray items)
|
||||
{
|
||||
var existing = old[category.Key]!.AsArray();
|
||||
for (var i = 0; i < items.Count; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
if (item is null) continue;
|
||||
var matches = Enumerable.Range(0, existing.Count).Where(j => Same(category.Key, existing[j], item)).ToList();
|
||||
int? match = matches.Count == 1 ? matches[0] : null;
|
||||
Add(category.Key + "." + i, category.Key, item, match is int index ? existing[index] : null, match, true);
|
||||
}
|
||||
}
|
||||
else Add(category.Key, category.Key, category.Value, old[category.Key], null, false);
|
||||
}
|
||||
return groups;
|
||||
|
||||
void Add(string id, string category, JsonNode item, JsonNode? existing, int? index, bool array)
|
||||
{
|
||||
var fields = new List<CvImportField>();
|
||||
void Walk(JsonNode? value, JsonNode? previous, string path)
|
||||
{
|
||||
if (value is JsonObject obj)
|
||||
{
|
||||
foreach (var property in obj) if (!Excluded.Contains(property.Key)) Walk(property.Value, (previous as JsonObject)?[property.Key], path + "." + property.Key);
|
||||
}
|
||||
else if (value is JsonArray list)
|
||||
{
|
||||
for (var j = 0; j < list.Count; j++)
|
||||
{
|
||||
var index = path == "contact.links" && previous is JsonArray links && list[j] is JsonObject link ? LinkIndex(links, link) : j;
|
||||
Walk(list[j], previous is JsonArray before && index is int matched && matched < before.Count ? before[matched] : null, path + "." + j);
|
||||
}
|
||||
}
|
||||
else if (value is JsonValue scalar)
|
||||
{
|
||||
var text = scalar.ToString();
|
||||
if (string.IsNullOrWhiteSpace(text) || (scalar.TryGetValue<bool>(out var boolean) && !boolean)) return;
|
||||
incoming.Metadata.Fields.TryGetValue(path, out var source);
|
||||
var label = Regex.Replace(path.Split('.').Last(), "([a-z])([A-Z])", "$1 $2");
|
||||
if (int.TryParse(label, out _)) label = "Item";
|
||||
label = char.ToUpperInvariant(label[0]) + label[1..];
|
||||
fields.Add(new(path, label, previous?.ToString(), text, source?.SourceSnippet, source?.SourcePage, source?.Method ?? "deterministic-source", "suggested"));
|
||||
}
|
||||
}
|
||||
Walk(item, existing, id);
|
||||
if (fields.Count > 0) groups.Add(new(new(id, category, category, array ? index.HasValue ? "Matched existing record" : "New record" : null, fields), item, category, index, array));
|
||||
}
|
||||
}
|
||||
|
||||
private static bool Same(string category, JsonNode? left, JsonNode right)
|
||||
{
|
||||
if (left is JsonValue && right is JsonValue) return string.Equals(left.ToString(), right.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
var keys = category switch
|
||||
{
|
||||
"jobs" => new[] { "company", "title", "start", "end" },
|
||||
"education" => new[] { "institution", "qualification", "start", "end" },
|
||||
"certifications" => new[] { "name", "issuer", "date" },
|
||||
"projects" => new[] { "name", "role", "start", "end" },
|
||||
"languages" => new[] { "name" },
|
||||
"otherSections" => new[] { "title" },
|
||||
_ => Array.Empty<string>()
|
||||
};
|
||||
return keys.Length > 0 && keys.All(k => !string.IsNullOrWhiteSpace(right[k]?.ToString()) && string.Equals(Clean(left?[k]?.ToString() ?? ""), Clean(right[k]!.ToString()), StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static int? LinkIndex(JsonArray links, JsonObject incoming)
|
||||
{
|
||||
var label = Clean(incoming["label"]?.ToString() ?? "");
|
||||
var matches = Enumerable.Range(0, links.Count).Where(i => label.Length > 0 && string.Equals(Clean(links[i]?["label"]?.ToString() ?? ""), label, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
return matches.Count == 1 ? matches[0] : null;
|
||||
}
|
||||
|
||||
public static StructuredCvProfile Apply(StructuredCvProfile current, StructuredCvProfile incoming, IReadOnlyList<CvImportDecision> decisions)
|
||||
{
|
||||
var maps = Maps(current, incoming);
|
||||
var known = maps.SelectMany(m => m.Review.Fields).ToDictionary(f => f.Id, StringComparer.Ordinal);
|
||||
if (decisions.Count > 2000 || decisions.Select(d => d.Id).Distinct(StringComparer.Ordinal).Count() != decisions.Count) throw new ArgumentException("Duplicate or excessive review decisions.");
|
||||
foreach (var d in decisions)
|
||||
if (!known.ContainsKey(d.Id) || d.Action is not ("keep" or "use" or "correct" or "add") || (d.Action == "correct" && (string.IsNullOrWhiteSpace(d.Value) || d.Value.Length > 8000 || d.Value.Any(c => char.IsControl(c) && c is not '\n' and not '\r' and not '\t')))) throw new ArgumentException("Invalid review decision.");
|
||||
var selected = decisions.Where(d => d.Action != "keep").ToDictionary(d => d.Id, StringComparer.Ordinal);
|
||||
var result = Node(current);
|
||||
var linkTargets = new Dictionary<int, int>();
|
||||
foreach (var map in maps)
|
||||
{
|
||||
var choices = map.Review.Fields.Where(f => selected.ContainsKey(f.Id)).ToList();
|
||||
if (choices.Count == 0) continue;
|
||||
JsonNode target;
|
||||
string targetPrefix;
|
||||
if (map.IsArray)
|
||||
{
|
||||
var list = result[map.Category]!.AsArray();
|
||||
var index = map.ExistingIndex ?? list.Count;
|
||||
if (!map.ExistingIndex.HasValue) list.Add(map.Incoming is JsonObject ? new JsonObject() : JsonValue.Create(""));
|
||||
target = list[index]!;
|
||||
targetPrefix = map.Category + "." + index;
|
||||
}
|
||||
else { target = result[map.Category]!; targetPrefix = map.Category; }
|
||||
foreach (var field in choices)
|
||||
{
|
||||
var decision = selected[field.Id];
|
||||
var text = decision.Action == "correct" ? decision.Value!.Trim() : field.ExtractedValue;
|
||||
var suffix = field.Id[map.Review.Id.Length..].TrimStart('.');
|
||||
if (map.Category == "contact" && suffix.StartsWith("links.", StringComparison.Ordinal))
|
||||
{
|
||||
var segments = suffix.Split('.');
|
||||
var incomingIndex = int.Parse(segments[1]);
|
||||
if (!linkTargets.TryGetValue(incomingIndex, out var targetIndex))
|
||||
{
|
||||
var links = result["contact"]!["links"]!.AsArray();
|
||||
targetIndex = LinkIndex(links, map.Incoming["links"]![incomingIndex]!.AsObject()) ?? links.Count;
|
||||
linkTargets[incomingIndex] = targetIndex;
|
||||
}
|
||||
suffix = $"links.{targetIndex}.{segments[2]}";
|
||||
}
|
||||
JsonNode value = JsonValue.Create(text)!;
|
||||
if (field.Id.EndsWith(".isCurrent", StringComparison.Ordinal))
|
||||
{
|
||||
if (!bool.TryParse(text, out var boolean)) throw new ArgumentException("isCurrent must be true or false.");
|
||||
value = JsonValue.Create(boolean)!;
|
||||
}
|
||||
if (suffix.Length == 0)
|
||||
{
|
||||
var list = result[map.Category]!.AsArray();
|
||||
list[int.Parse(targetPrefix.Split('.').Last())] = value;
|
||||
}
|
||||
else Set(target, suffix.Split('.'), value);
|
||||
var path = targetPrefix + (suffix.Length == 0 ? "" : "." + suffix);
|
||||
incoming.Metadata.Fields.TryGetValue(field.Id, out var source);
|
||||
var metadataNode = result["metadata"]!["fields"]!.AsObject();
|
||||
metadataNode[path] = JsonSerializer.SerializeToNode(new StructuredCvFieldMetadata { Method = decision.Action == "correct" ? "user-correction" : source?.Method ?? "deterministic-source", SourceSnippet = source?.SourceSnippet, SourcePage = source?.SourcePage, SourceBlockId = source?.SourceBlockId, ReviewState = decision.Action == "correct" ? "corrected" : "accepted", LastUpdatedAtUtc = DateTimeOffset.UtcNow }, Json);
|
||||
}
|
||||
}
|
||||
return result.Deserialize<StructuredCvProfile>(Json)!;
|
||||
}
|
||||
|
||||
private static void Set(JsonNode target, string[] path, JsonNode value)
|
||||
{
|
||||
for (var i = 0; i < path.Length - 1; i++)
|
||||
{
|
||||
var key = path[i];
|
||||
var arrayNext = int.TryParse(path[i + 1], out _);
|
||||
if (target is JsonArray list)
|
||||
{
|
||||
var index = int.Parse(key);
|
||||
while (list.Count <= index) list.Add(null);
|
||||
list[index] ??= arrayNext ? new JsonArray() : new JsonObject();
|
||||
target = list[index]!;
|
||||
}
|
||||
else { target[key] ??= arrayNext ? new JsonArray() : new JsonObject(); target = target[key]!; }
|
||||
}
|
||||
if (target is JsonArray array)
|
||||
{
|
||||
var index = int.Parse(path[^1]);
|
||||
// Individual approved list entries append instead of erasing user-only entries.
|
||||
if (!array.Any(v => v?.ToString() == value.ToString())) array.Add(value);
|
||||
}
|
||||
else target[path[^1]] = value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
using JobTrackerApi.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace JobTrackerApi.Services;
|
||||
|
||||
public sealed class CvImportRetentionHostedService(IServiceScopeFactory scopes, AppPaths paths, ILogger<CvImportRetentionHostedService> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromHours(1));
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var scope = scopes.CreateAsyncScope();
|
||||
await SweepAsync(scope.ServiceProvider.GetRequiredService<JobTrackerContext>(), paths, DateTimeOffset.UtcNow, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
catch (Exception) { logger.LogWarning("CV import retention could not complete; it will retry on the next sweep."); }
|
||||
} while (await timer.WaitForNextTickAsync(stoppingToken));
|
||||
}
|
||||
|
||||
public static async Task<int> SweepAsync(JobTrackerContext db, AppPaths paths, DateTimeOffset now, CancellationToken token)
|
||||
{
|
||||
var runs = await db.CvExtractionRuns.IgnoreQueryFilters().Include(r => r.Artifact)
|
||||
.Where(r => r.ParserVersion == "docling-v1" && r.Artifact != null && r.Artifact.StoragePath != "")
|
||||
.OrderBy(r => r.Id).Take(1000).ToListAsync(token);
|
||||
var root = Path.GetFullPath(paths.CvArtifactsRoot) + Path.DirectorySeparatorChar;
|
||||
var removed = 0;
|
||||
foreach (var run in runs)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
if (run.StartedAtUtc > now.AddDays(-7) && (!(run.Status is "cancelled" or "failed" or "discarded") || run.StartedAtUtc > now.AddDays(-1))) continue;
|
||||
var artifact = run.Artifact!;
|
||||
if (artifact.OwnerUserId != run.OwnerUserId) continue;
|
||||
var path = Path.GetFullPath(artifact.StoragePath);
|
||||
if (!path.StartsWith(root, OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal)) continue;
|
||||
// Revoke the operation before deleting its source; a late worker cannot publish a result.
|
||||
var subject = run.Id.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
var operations = await db.UserOperations.IgnoreQueryFilters().Where(o => o.OwnerUserId == run.OwnerUserId && o.SubjectType == CvProcessingQueue.SubjectType && o.SubjectId == subject).ToListAsync(token);
|
||||
foreach (var operation in operations.Where(o => !Models.OperationStatuses.IsTerminal(o.Status)))
|
||||
{
|
||||
operation.Status = Models.OperationStatuses.Cancelled;
|
||||
operation.CancellationRequestedAtUtc = now.UtcDateTime;
|
||||
operation.CompletedAtUtc = now.UtcDateTime;
|
||||
}
|
||||
await db.SaveChangesAsync(token);
|
||||
try
|
||||
{
|
||||
if (File.Exists(path) && (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0) continue;
|
||||
File.Delete(path);
|
||||
}
|
||||
catch (IOException) { continue; }
|
||||
catch (UnauthorizedAccessException) { continue; }
|
||||
artifact.StoragePath = "";
|
||||
run.DocumentJson = null;
|
||||
run.RawExtractedText = null;
|
||||
run.NormalizedText = null;
|
||||
if (run.Status != "applied")
|
||||
{
|
||||
run.StructuredProfileJson = null;
|
||||
if (run.Status is "pending_review" or "queued" or "running") run.Status = "expired";
|
||||
run.CompletedAtUtc ??= now;
|
||||
run.ErrorMessage = "The source document expired. Upload it again to start a new review.";
|
||||
}
|
||||
removed++;
|
||||
}
|
||||
await db.SaveChangesAsync(token);
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
@@ -64,7 +64,7 @@ public sealed class CvProcessingOperationHandler : IAiOperationHandler
|
||||
try
|
||||
{
|
||||
outcome = await services.GetRequiredService<ProfileCvController>()
|
||||
.ProcessQueuedRunAsync(runId, cancellationToken);
|
||||
.ProcessQueuedRunAsync(runId, cancellationToken, context);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
@@ -109,7 +109,7 @@ public sealed class CvProcessingOperationHandler : IAiOperationHandler
|
||||
}
|
||||
|
||||
return new AiOperationExecutionResult(
|
||||
$"/api/profile-cv/runs/{runId}/diff",
|
||||
$"/api/profile-cv/runs/{runId}/{(await services.GetRequiredService<JobTrackerContext>().CvExtractionRuns.Where(r => r.Id == runId).Select(r => r.ParserVersion).FirstAsync(cancellationToken) == ProfileCvController.DocumentParserVersion ? "review" : "diff")}",
|
||||
outcome.Provider,
|
||||
outcome.Model,
|
||||
outcome.RouteReason ?? "cv_pipeline");
|
||||
|
||||
@@ -39,6 +39,9 @@ public sealed class MeteredSummarizerService(
|
||||
=> MeterAsync("synchronous.generate", instruction + "\n\n" + text, maxLength,
|
||||
ct => inner.GenerateSectionWithMetadataAsync(instruction, text, outputMode, maxLength, minLength, ct), cancellationToken);
|
||||
|
||||
public Task<CvImportDocument?> ExtractCvDocumentAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default)
|
||||
=> inner.ExtractCvDocumentAsync(stream, fileName, contentType, cancellationToken);
|
||||
|
||||
public Task<AiTextExtractionResult?> ExtractTextAsync(
|
||||
Stream stream,
|
||||
string fileName,
|
||||
|
||||
@@ -94,6 +94,8 @@ namespace JobTrackerApi.Services
|
||||
|
||||
public interface ISummarizerService : IAiService
|
||||
{
|
||||
Task<CvImportDocument?> ExtractCvDocumentAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult<CvImportDocument?>(null);
|
||||
new Task<string?> SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40);
|
||||
|
||||
// Older providers cannot approve factual changes without implementing this review boundary.
|
||||
@@ -492,6 +494,32 @@ namespace JobTrackerApi.Services
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<CvImportDocument?> ExtractCvDocumentAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var form = new MultipartFormDataContent();
|
||||
using var file = new StreamContent(stream);
|
||||
file.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType ?? "application/octet-stream");
|
||||
form.Add(file, "file", Path.GetFileName(fileName));
|
||||
using var response = await _httpFactory.CreateClient("ai-service").PostAsync("/cv-import/extract", form, cancellationToken);
|
||||
if (!response.IsSuccessStatusCode) throw new AiGenerationException("cv_extraction_failed", "Document extraction is unavailable or the document could not be parsed.", (int)response.StatusCode >= 500);
|
||||
await using var body = await response.Content.ReadAsStreamAsync(cancellationToken);
|
||||
using var bounded = new MemoryStream();
|
||||
var buffer = new byte[8192];
|
||||
int read;
|
||||
while ((read = await body.ReadAsync(buffer, cancellationToken)) > 0)
|
||||
{
|
||||
if (bounded.Length + read > 2 * 1024 * 1024) throw new InvalidOperationException("Document extraction response exceeded its limit.");
|
||||
await bounded.WriteAsync(buffer.AsMemory(0, read), cancellationToken);
|
||||
}
|
||||
using var json = JsonDocument.Parse(bounded.ToArray());
|
||||
var root = json.RootElement;
|
||||
var text = root.GetProperty("text").GetString() ?? "";
|
||||
var blocks = root.GetProperty("blocks").EnumerateArray().Select(b => new CvImportBlock(b.GetProperty("id").GetString() ?? "", b.GetProperty("text").GetString() ?? "", b.GetProperty("kind").GetString() ?? "text", b.TryGetProperty("page", out var page) && page.ValueKind == JsonValueKind.Number ? page.GetInt32() : null)).ToList();
|
||||
var warnings = root.GetProperty("warnings").EnumerateArray().Select(w => w.GetString() ?? "").ToList();
|
||||
if (string.IsNullOrWhiteSpace(text) || text.Length > 500000 || blocks.Count is 0 or > 10000 || blocks.Any(b => b.Id.Length is 0 or > 128 || b.Text.Length > 100000 || b.Page is < 1 or > 1000) || blocks.Select(b => b.Id).Distinct().Count() != blocks.Count || warnings.Count > 100 || warnings.Any(w => w.Length > 1000)) throw new InvalidOperationException("Document extraction returned invalid evidence.");
|
||||
return new CvImportDocument(text, root.GetProperty("ocr_used").GetBoolean(), root.GetProperty("content_type").GetString(), root.TryGetProperty("page_count", out var pages) && pages.ValueKind == JsonValueKind.Number ? pages.GetInt32() : null, text.Length, Path.GetFileName(fileName), blocks, warnings, root.GetProperty("method").GetString() ?? "docling");
|
||||
}
|
||||
|
||||
public async Task<AiTextExtractionResult?> ExtractTextAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
|
||||
@@ -169,8 +169,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
|
||||
.ExecuteUpdateAsync(setters => setters
|
||||
.SetProperty(operation => operation.LeaseExpiresAtUtc, now.Add(leaseDuration))
|
||||
.SetProperty(operation => operation.LastHeartbeatAtUtc, now)
|
||||
.SetProperty(operation => operation.ProgressStage, progressStage)
|
||||
.SetProperty(operation => operation.ProgressPercent, progressPercent),
|
||||
.SetProperty(operation => operation.ProgressStage, operation => progressStage ?? operation.ProgressStage)
|
||||
.SetProperty(operation => operation.ProgressPercent, operation => progressPercent ?? operation.ProgressPercent),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user