cesnimda
248 lines
19 KiB
C#
248 lines
19 KiB
C#
using System.IO.Compression;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using JobTrackerApi.Models;
|
|
using JobTrackerApi.Services;
|
|
using Microsoft.AspNetCore.Authorization;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Microsoft.EntityFrameworkCore;
|
|
|
|
namespace JobTrackerApi.Controllers;
|
|
|
|
public sealed partial class ProfileCvController
|
|
{
|
|
public const string DocumentParserVersion = "docling-v1";
|
|
public sealed record ApplyReviewedRequest(int? ExpectedVersion, IReadOnlyList<CvImportDecision>? Decisions, IReadOnlyList<CvImportRecordMapping>? Mappings = null);
|
|
private sealed record AppliedReview(string Hash, int Revision, StructuredCvProfile StructuredCv, IReadOnlyList<CvImportDecision>? Decisions = null, IReadOnlyList<CvImportRecordMapping>? Mappings = null);
|
|
|
|
private async Task ProcessDocumentImportAsync(CvExtractionRun run, AiOperationExecutionContext? context, CancellationToken cancellationToken)
|
|
{
|
|
async Task Stage(string stage)
|
|
{
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
if (context is null) return;
|
|
var lease = context.Lease;
|
|
var updated = await new UserOperationStore(_db, TimeProvider.System)
|
|
.ReportStageAsync(lease.OperationId, lease.LeaseToken, stage, cancellationToken);
|
|
if (updated != 1) throw new OperationCanceledException("The document processing lease is no longer active.", cancellationToken);
|
|
}
|
|
await Stage("cv_validating_file");
|
|
var artifact = await _db.CvUploadArtifacts.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(a => a.Id == run.ArtifactId && a.OwnerUserId == run.OwnerUserId, cancellationToken);
|
|
if (artifact is null || !System.IO.File.Exists(artifact.StoragePath)) throw new InvalidOperationException("The stored source document is unavailable.");
|
|
await using var stream = System.IO.File.OpenRead(artifact.StoragePath);
|
|
if (!string.Equals(Convert.ToHexString(await SHA256.HashDataAsync(stream, cancellationToken)), artifact.Sha256, StringComparison.OrdinalIgnoreCase)) throw new InvalidOperationException("The source document changed. Upload it again.");
|
|
stream.Position = 0;
|
|
await Stage("cv_extracting");
|
|
var document = await _aiService.ExtractCvDocumentAsync(stream, artifact.OriginalFileName, artifact.MimeType, cancellationToken)
|
|
?? throw new InvalidOperationException("Document extraction is unavailable. The source was retained for retry.");
|
|
await Stage("cv_identifying_sections");
|
|
var blocks = document.Blocks;
|
|
await Stage("cv_analysing");
|
|
var candidate = CvDocumentInterpreter.Parse(blocks);
|
|
await Stage("cv_validating_details");
|
|
var profile = CvDocumentReview.Ground(candidate, document);
|
|
foreach (var field in profile.Metadata.Fields.Values)
|
|
{
|
|
field.SourceArtifactId = artifact.Id;
|
|
field.SourceExtractionRunId = run.Id;
|
|
field.SourceFileSha256 = artifact.Sha256;
|
|
}
|
|
await Stage("cv_preparing_review");
|
|
await Stage("cv_review_ready");
|
|
cancellationToken.ThrowIfCancellationRequested();
|
|
var documentJson = JsonSerializer.Serialize(document);
|
|
var profileJson = StructuredCvProfileJson.SerializePersisted(profile);
|
|
if (context is null)
|
|
{
|
|
run.DocumentJson = documentJson;
|
|
await CompleteQueuedRunForReviewAsync(run, document.Text, document.Text, profile, cancellationToken);
|
|
return;
|
|
}
|
|
var currentLease = context.Lease;
|
|
var completedAt = DateTimeOffset.UtcNow;
|
|
var published = await _db.CvExtractionRuns.Where(r => r.Id == run.Id && r.OwnerUserId == run.OwnerUserId && r.Status == "running" && _db.UserOperations.Any(o => o.Id == currentLease.OperationId && o.LeaseToken == currentLease.LeaseToken && o.Status == OperationStatuses.Running && o.CancellationRequestedAtUtc == null && o.LeaseExpiresAtUtc > DateTime.UtcNow))
|
|
.ExecuteUpdateAsync(s => s.SetProperty(r => r.DocumentJson, documentJson).SetProperty(r => r.RawExtractedText, document.Text).SetProperty(r => r.NormalizedText, document.Text).SetProperty(r => r.StructuredProfileJson, profileJson).SetProperty(r => r.Status, "pending_review").SetProperty(r => r.CompletedAtUtc, completedAt), cancellationToken);
|
|
if (published != 1) throw new OperationCanceledException("The document processing lease was lost before publication.", cancellationToken);
|
|
await _db.Entry(run).ReloadAsync(cancellationToken);
|
|
await PruneExtractionRunsAsync(run.OwnerUserId, cancellationToken);
|
|
}
|
|
|
|
[HttpPost("import")]
|
|
[Authorize(Policy = ProEntitlement.Policy)]
|
|
[RequestSizeLimit(MaxFileSizeBytes + 65536)]
|
|
public async Task<IActionResult> Import([FromForm] IFormFile file)
|
|
{
|
|
var user = await _users.GetUserAsync(User);
|
|
if (user is null) return Unauthorized();
|
|
if (file is null || file.Length is <= 0 or > MaxFileSizeBytes) return BadRequest("Select a PDF or DOCX CV under 5 MB.");
|
|
var extension = Path.GetExtension(file.FileName).ToLowerInvariant();
|
|
if (extension is not (".pdf" or ".docx")) return BadRequest("Select a PDF or DOCX CV.");
|
|
try { await ValidateImportAsync(file, extension, HttpContext.RequestAborted); }
|
|
catch (InvalidDataException) { return BadRequest("The document signature or archive is invalid or unsupported."); }
|
|
if (await _db.CvExtractionRuns.CountAsync(r => r.OwnerUserId == user.Id && r.Status == "pending_review", HttpContext.RequestAborted) >= ExtractionRunRetentionCount) return Conflict("Review or discard an existing import before uploading another document.");
|
|
if (await _db.CvUploadArtifacts.CountAsync(a => a.OwnerUserId == user.Id && a.StoragePath != "", HttpContext.RequestAborted) >= 100) return Conflict("Remove or finish existing imports before uploading more documents.");
|
|
var artifact = await SaveUploadArtifactAsync(user, file, HttpContext.RequestAborted);
|
|
artifact.OriginalFileName = Path.GetFileName(file.FileName.Replace('\\', '/'));
|
|
artifact.MimeType = extension == ".pdf" ? "application/pdf" : "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
|
|
var stored = false;
|
|
try
|
|
{
|
|
var active = await FindActiveRunAsync(user.Id, "document_import", null, artifact.Sha256, HttpContext.RequestAborted);
|
|
if (active is not null) { TryDeleteCvArtifactFile(artifact.StoragePath); return await EnqueueRunAsync(active, HttpContext.RequestAborted); }
|
|
var run = new CvExtractionRun { OwnerUserId = user.Id, Artifact = artifact, Trigger = "document_import", ParserVersion = DocumentParserVersion, NormalizerVersion = "source-grounded-v1", LlmPromptVersion = "none", Status = "queued" };
|
|
_db.CvExtractionRuns.Add(run);
|
|
await _db.SaveChangesAsync(HttpContext.RequestAborted);
|
|
stored = true;
|
|
return await EnqueueRunAsync(run, HttpContext.RequestAborted);
|
|
}
|
|
catch { if (!stored) TryDeleteCvArtifactFile(artifact.StoragePath); throw; }
|
|
}
|
|
|
|
private static async Task ValidateImportAsync(IFormFile file, string extension, CancellationToken cancellationToken)
|
|
{
|
|
await using var source = file.OpenReadStream();
|
|
using var bytes = new MemoryStream();
|
|
var buffer = new byte[8192];
|
|
int read;
|
|
while ((read = await source.ReadAsync(buffer, cancellationToken)) > 0)
|
|
{
|
|
if (bytes.Length + read > MaxFileSizeBytes) throw new InvalidDataException();
|
|
bytes.Write(buffer, 0, read);
|
|
}
|
|
var data = bytes.ToArray();
|
|
if (extension == ".pdf")
|
|
{
|
|
if (data.Length < 8 || !data.AsSpan(0, 5).SequenceEqual("%PDF-"u8)) throw new InvalidDataException();
|
|
var tokens = Encoding.Latin1.GetString(data);
|
|
if (new[] { "/JavaScript", "/JS", "/Launch", "/OpenAction", "/EmbeddedFile", "/Encrypt" }.Any(tokens.Contains)) throw new InvalidDataException();
|
|
return;
|
|
}
|
|
bytes.Position = 0;
|
|
try
|
|
{
|
|
using var archive = new ZipArchive(bytes, ZipArchiveMode.Read, true);
|
|
if (archive.Entries.Count is 0 or > 2048 || archive.GetEntry("word/document.xml") is null || archive.GetEntry("[Content_Types].xml") is null) throw new InvalidDataException();
|
|
long expanded = 0;
|
|
foreach (var entry in archive.Entries)
|
|
{
|
|
expanded += entry.Length;
|
|
if (entry.FullName.Contains("..") || entry.FullName.StartsWith('/') || entry.FullName.Contains('\\') || entry.Length > 10 * 1024 * 1024 || expanded > 25 * 1024 * 1024 || entry.Length > 200 * Math.Max(1, entry.CompressedLength) || entry.FullName.EndsWith("vbaProject.bin", StringComparison.OrdinalIgnoreCase)) throw new InvalidDataException();
|
|
if (entry.FullName.EndsWith(".rels", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
using var reader = new StreamReader(entry.Open());
|
|
var relationships = await reader.ReadToEndAsync(cancellationToken);
|
|
// Hyperlinks remain inert text; linked templates/OLE/media are refused.
|
|
using var xmlReader = System.Xml.XmlReader.Create(new StringReader(relationships), new System.Xml.XmlReaderSettings { DtdProcessing = System.Xml.DtdProcessing.Prohibit, XmlResolver = null, MaxCharactersInDocument = 2 * 1024 * 1024 });
|
|
var xml = System.Xml.Linq.XDocument.Load(xmlReader);
|
|
if (xml.Descendants().Any(e => string.Equals((string?)e.Attribute("TargetMode"), "External", StringComparison.OrdinalIgnoreCase) && !((string?)e.Attribute("Type") ?? "").EndsWith("/hyperlink", StringComparison.Ordinal))) throw new InvalidDataException();
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex) when (ex is IOException or System.Xml.XmlException or NotSupportedException) { throw new InvalidDataException("Invalid document archive.", ex); }
|
|
}
|
|
|
|
[NonAction]
|
|
public static StructuredCvProfile ParseImportDocument(CvImportDocument document)
|
|
{
|
|
var candidate = CvDocumentInterpreter.Parse(document.Blocks);
|
|
return CvDocumentReview.Ground(candidate, document);
|
|
}
|
|
|
|
[HttpGet("runs/{id:int}/review")]
|
|
public async Task<IActionResult> GetImportReview(int id)
|
|
{
|
|
var user = await _users.GetUserAsync(User);
|
|
if (user is null) return Unauthorized();
|
|
var run = await _db.CvExtractionRuns.AsNoTracking().Include(r => r.Artifact).FirstOrDefaultAsync(r => r.Id == id && r.OwnerUserId == user.Id, HttpContext.RequestAborted);
|
|
if (run is null) return NotFound();
|
|
if (run.ParserVersion != DocumentParserVersion || string.IsNullOrWhiteSpace(run.StructuredProfileJson)) return Conflict("This import is not ready for review.");
|
|
var snapshot = await _careerProfileService.LoadSnapshotAsync(user.Id, HttpContext.RequestAborted);
|
|
var document = JsonSerializer.Deserialize<CvImportDocument>(run.DocumentJson ?? "null");
|
|
return Ok(new CvImportReview(run.Id, run.Status, snapshot.Revision, run.Artifact?.OriginalFileName, $"/api/profile-cv/runs/{run.Id}/source", run.RawExtractedText, document?.Warnings ?? Array.Empty<string>(), CvDocumentReview.Build(snapshot.Profile, StructuredCvProfileJson.DeserializePersisted(run.StructuredProfileJson)), document?.Blocks ?? Array.Empty<CvImportBlock>(), CvImportMapping.Catalog(snapshot.Profile), CvImportMapping.Targets(snapshot.Profile)));
|
|
}
|
|
|
|
[HttpGet("runs/{id:int}/source")]
|
|
public async Task<IActionResult> GetImportSource(int id)
|
|
{
|
|
var user = await _users.GetUserAsync(User);
|
|
if (user is null) return Unauthorized();
|
|
var run = await _db.CvExtractionRuns.AsNoTracking().Include(r => r.Artifact).FirstOrDefaultAsync(r => r.Id == id && r.OwnerUserId == user.Id, HttpContext.RequestAborted);
|
|
var artifact = run?.Artifact;
|
|
if (artifact is null || artifact.OwnerUserId != user.Id || !System.IO.File.Exists(artifact.StoragePath)) return NotFound();
|
|
Response.Headers.CacheControl = "private, no-store";
|
|
Response.Headers["X-Content-Type-Options"] = "nosniff";
|
|
return PhysicalFile(artifact.StoragePath, artifact.MimeType, artifact.OriginalFileName);
|
|
}
|
|
|
|
[HttpPost("runs/{id:int}/apply-reviewed")]
|
|
public async Task<IActionResult> ApplyReviewed(int id, [FromBody] ApplyReviewedRequest? request)
|
|
{
|
|
var user = await _users.GetUserAsync(User);
|
|
if (user is null) return Unauthorized();
|
|
if (request?.ExpectedVersion is null or < 0 || request.Decisions is null || request.Decisions.Any(d => d is null || d.Id is null || d.Action is null)) return BadRequest("expectedVersion and review decisions are required.");
|
|
var mappings = request.Mappings ?? Array.Empty<CvImportRecordMapping>();
|
|
if (mappings.Count > 25 || mappings.Any(m => m is null)) return BadRequest("Invalid or excessive record mappings.");
|
|
// Preserve hashes for old requests and already-applied historical reviews.
|
|
var hashBytes = mappings.Count == 0 ? JsonSerializer.SerializeToUtf8Bytes(new { request.ExpectedVersion, Decisions = request.Decisions.OrderBy(d => d.Id, StringComparer.Ordinal) })
|
|
: JsonSerializer.SerializeToUtf8Bytes(new { request.ExpectedVersion, Decisions = request.Decisions.OrderBy(d => d.Id, StringComparer.Ordinal), Mappings = mappings });
|
|
var hash = Convert.ToHexString(SHA256.HashData(hashBytes));
|
|
try
|
|
{
|
|
return await _careerProfileService.ExecuteAtomicAsync<IActionResult>(async () =>
|
|
{
|
|
var run = await _db.CvExtractionRuns.Include(r => r.Artifact).FirstOrDefaultAsync(r => r.Id == id && r.OwnerUserId == user.Id, HttpContext.RequestAborted);
|
|
if (run is null) return NotFound();
|
|
if (run.ParserVersion != DocumentParserVersion) return Conflict("Use legacy review for this run.");
|
|
if (run.Status == "applied")
|
|
{
|
|
var prior = JsonSerializer.Deserialize<AppliedReview>(run.ReviewJson ?? "null");
|
|
var sameMappings = mappings.Count == 0 || JsonSerializer.Serialize(prior?.Mappings) == JsonSerializer.Serialize(mappings);
|
|
var identical = prior?.Hash == hash && sameMappings;
|
|
// Old mapping receipts sorted IDs; accept their exact original order only.
|
|
if (!identical && prior is not null && mappings.Count > 0 && sameMappings)
|
|
{
|
|
var legacyHash = Convert.ToHexString(SHA256.HashData(JsonSerializer.SerializeToUtf8Bytes(new { request.ExpectedVersion, Decisions = request.Decisions.OrderBy(d => d.Id, StringComparer.Ordinal), Mappings = mappings.OrderBy(m => m.ClientId, StringComparer.Ordinal) })));
|
|
identical = prior.Hash == legacyHash;
|
|
}
|
|
return identical ? Ok(new { runId = id, status = "applied", revision = prior!.Revision, structuredCv = prior.StructuredCv }) : Conflict("This import has already been applied with different decisions.");
|
|
}
|
|
if (run.Status != "pending_review" || string.IsNullOrWhiteSpace(run.StructuredProfileJson)) return Conflict("This import is not awaiting review.");
|
|
var snapshot = await _careerProfileService.LoadSnapshotAsync(user.Id, HttpContext.RequestAborted);
|
|
if (snapshot.Revision != request.ExpectedVersion) throw new DbUpdateConcurrencyException();
|
|
CvImportMapping.ValidateTargets(snapshot.Profile, mappings);
|
|
var merged = CvDocumentReview.Apply(snapshot.Profile, StructuredCvProfileJson.DeserializePersisted(run.StructuredProfileJson), request.Decisions);
|
|
if (mappings.Count > 0)
|
|
{
|
|
var document = JsonSerializer.Deserialize<CvImportDocument>(run.DocumentJson ?? "null");
|
|
if (document is null || (run.Artifact is not null && run.Artifact.OwnerUserId != user.Id)) return BadRequest("The source evidence is unavailable.");
|
|
merged = CvImportMapping.Apply(merged, mappings, document, run.ArtifactId, run.Id, run.Artifact?.Sha256);
|
|
}
|
|
if (!request.Decisions.Any(d => d.Action != "keep") && mappings.Count == 0) return BadRequest("No imported fields were selected. Discard this import to keep your current profile.");
|
|
var validation = CareerProfileValidator.Validate(merged);
|
|
if (validation is not null || StructuredCvProfileJson.SerializePersisted(merged).Length > 500000) return BadRequest(validation ?? "The reviewed profile exceeds its size limit.");
|
|
merged.Metadata.ProfileVersion = snapshot.Revision + 1;
|
|
merged.Metadata.AppliedExtractionRunId = run.Id;
|
|
merged.Metadata.UpdatedAtUtc = DateTimeOffset.UtcNow;
|
|
await _careerProfileService.SaveVersionAsync(user.Id, merged, "document_import:reviewed", request.ExpectedVersion.Value, HttpContext.RequestAborted);
|
|
user.ProfileCvStructureJson = StructuredCvProfileJson.SerializePersisted(merged);
|
|
user.CurrentCvExtractionRunId = run.Id;
|
|
user.CurrentCvProfileVersion = merged.Metadata.ProfileVersion;
|
|
user.CurrentCvUploadArtifactId = run.ArtifactId;
|
|
run.Status = "applied";
|
|
run.AppliedAtUtc = DateTimeOffset.UtcNow;
|
|
run.ReviewJson = JsonSerializer.Serialize(new AppliedReview(hash, snapshot.Revision + 1, merged, request.Decisions, request.Mappings));
|
|
var update = await _users.UpdateAsync(user);
|
|
if (!update.Succeeded) throw new CareerProjectionException(string.Join("; ", update.Errors.Select(e => e.Description)));
|
|
await _db.SaveChangesAsync(HttpContext.RequestAborted);
|
|
return Ok(new { runId = id, run.Status, revision = snapshot.Revision + 1, structuredCv = merged });
|
|
}, HttpContext.RequestAborted);
|
|
}
|
|
catch (DbUpdateConcurrencyException) { return Conflict(new { code = "revision_conflict", message = "The career profile changed. Refresh the comparison and review your decisions." }); }
|
|
// MariaDB can report a changed run as a snapshot conflict before EF sees zero affected rows.
|
|
catch (DbUpdateException ex) when (ex.InnerException is MySqlConnector.MySqlException { Number: 1020 }) { return Conflict(new { code = "revision_conflict", message = "The import changed. Refresh the comparison and review your decisions." }); }
|
|
catch (ArgumentException ex) { return BadRequest(ex.Message); }
|
|
catch (CareerProjectionException ex) { return BadRequest(ex.Message); }
|
|
}
|
|
}
|