feat(cv-import): ground reviewed facts in revisioned career profiles
This commit is contained in:
cesnimda
@@ -241,9 +241,10 @@ public sealed class AiOperationWorker(
|
||||
execution.Cancel();
|
||||
return;
|
||||
}
|
||||
await store.HeartbeatAsync(lease.OperationId, lease.LeaseToken,
|
||||
var renewed = await store.HeartbeatAsync(lease.OperationId, lease.LeaseToken,
|
||||
TimeSpan.FromSeconds(Math.Clamp(configuration.GetValue("AiQueue:LeaseSeconds", 120), 30, 1800)),
|
||||
"processing", null, cancellationToken);
|
||||
null, null, cancellationToken);
|
||||
if (renewed != 1) { execution.Cancel(); return; }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
using System.Text.RegularExpressions;
|
||||
using JobTrackerApi.Models;
|
||||
|
||||
namespace JobTrackerApi.Services;
|
||||
|
||||
// Structural evidence is interpreted before any profile is proposed; ambiguous source stays reviewable.
|
||||
public static class CvDocumentInterpreter
|
||||
{
|
||||
private static readonly Dictionary<string, string> Headings = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["contact"] = "contact", ["kontakt"] = "contact", ["contact details"] = "contact",
|
||||
["professional summary"] = "summary", ["summary"] = "summary", ["profile"] = "summary", ["profil"] = "summary", ["sammendrag"] = "summary",
|
||||
["work experience"] = "jobs", ["experience"] = "jobs", ["employment history"] = "jobs", ["professional experience"] = "jobs", ["arbeidserfaring"] = "jobs", ["yrkeserfaring"] = "jobs", ["erfaring"] = "jobs",
|
||||
["education"] = "education", ["utdanning"] = "education", ["qualifications"] = "education",
|
||||
["skills"] = "skills", ["technical skills"] = "skills", ["core skills"] = "skills", ["ferdigheter"] = "skills", ["kompetanse"] = "skills", ["tekniske ferdigheter"] = "skills",
|
||||
["languages"] = "languages", ["språk"] = "languages", ["certifications"] = "certifications", ["sertifiseringer"] = "certifications",
|
||||
["projects"] = "projects", ["prosjekter"] = "projects", ["interests"] = "interests", ["interesser"] = "interests",
|
||||
["awards"] = "awards", ["publications"] = "publications", ["organisations"] = "organisations", ["organizations"] = "organisations", ["references"] = "references", ["referanser"] = "references",
|
||||
["courses"] = "Courses", ["kurs"] = "Kurs", ["volunteer experience"] = "Volunteer Experience", ["notes"] = "Notes",
|
||||
};
|
||||
|
||||
public static StructuredCvProfile Parse(IReadOnlyList<CvImportBlock> blocks)
|
||||
{
|
||||
var profile = new StructuredCvProfile();
|
||||
var section = "contact";
|
||||
var sectionTitle = "Contact";
|
||||
var unassigned = new List<string>();
|
||||
void Unknown(string line)
|
||||
{
|
||||
var other = profile.OtherSections.FirstOrDefault(s => s.Title == sectionTitle);
|
||||
if (other is null) { other = new() { Title = sectionTitle }; profile.OtherSections.Add(other); }
|
||||
other.Items.Add(line);
|
||||
}
|
||||
foreach (var block in blocks)
|
||||
foreach (var source in block.Text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var line = source.Trim().TrimStart('#', '•').Trim();
|
||||
if (line.StartsWith('|')) line = line.Trim('|').Trim();
|
||||
if (Regex.IsMatch(line, @"^[|:\s-]+$")) continue;
|
||||
var heading = line.TrimEnd(':');
|
||||
if (Headings.TryGetValue(heading, out var named)) { section = named; sectionTitle = heading; continue; }
|
||||
if (block.Kind == "section_header") { section = "custom"; sectionTitle = heading; continue; }
|
||||
var contact = Regex.Match(line, @"^(name|navn|full name|email|e-post|phone|telefon|location|sted|headline|title|website|nettside|linkedin|github|portfolio)\s*:\s*(.+)$", RegexOptions.IgnoreCase);
|
||||
if (section == "contact" && contact.Success)
|
||||
{
|
||||
var value = contact.Groups[2].Value.Trim();
|
||||
switch (contact.Groups[1].Value.ToLowerInvariant())
|
||||
{
|
||||
case "name": case "navn": case "full name": profile.Contact.FullName = value; break;
|
||||
case "email": case "e-post": profile.Contact.Email = value; break;
|
||||
case "phone": case "telefon": profile.Contact.Phone = value; break;
|
||||
case "location": case "sted": profile.Contact.Location = value; break;
|
||||
case "headline": case "title": profile.Contact.Headline = value; break;
|
||||
case "website": case "nettside": profile.Contact.Website = value; break;
|
||||
case "linkedin": profile.Contact.LinkedIn = value; break;
|
||||
case "github": profile.Contact.GitHub = value; break;
|
||||
case "portfolio": profile.Contact.Links.Add(new() { Label = contact.Groups[1].Value, Url = value }); break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
var parts = line.Split('|', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
switch (section)
|
||||
{
|
||||
case "contact":
|
||||
var email = Regex.Match(line, @"[\w.+-]+@[\w.-]+\.[\p{L}]{2,}");
|
||||
if (email.Success) profile.Contact.Email ??= email.Value;
|
||||
else if (block.Kind == "title" && profile.Contact.FullName is null) profile.Contact.FullName = line;
|
||||
else unassigned.Add(line);
|
||||
break;
|
||||
case "summary": profile.Summary.Add(line); break;
|
||||
case "skills": profile.Skills.AddRange(Regex.Split(line, @"\s*[,;•|]\s*").Where(s => s.Length > 0)); break;
|
||||
case "languages":
|
||||
var language = line.Split(':', 2, StringSplitOptions.TrimEntries);
|
||||
profile.Languages.Add(new() { Name = language[0], Level = language.Length == 2 ? language[1] : null }); break;
|
||||
case "jobs":
|
||||
if (parts.Length >= 2)
|
||||
{
|
||||
var (start, end) = Dates(parts.ElementAtOrDefault(2));
|
||||
profile.Jobs.Add(new() { Title = parts[0], Company = parts[1], Start = start, End = end, IsCurrent = end is not null && Regex.IsMatch(end, @"^(present|current|now|nå|dags dato|pågående)$", RegexOptions.IgnoreCase) });
|
||||
}
|
||||
else if (profile.Jobs.Count > 0) profile.Jobs[^1].Bullets.Add(line.TrimStart('-', ' '));
|
||||
else Unknown(line);
|
||||
break;
|
||||
case "education":
|
||||
if (parts.Length >= 2)
|
||||
{
|
||||
var (start, end) = Dates(parts.ElementAtOrDefault(2));
|
||||
profile.Education.Add(new() { Qualification = parts[0], Institution = parts[1], Start = start, End = end });
|
||||
}
|
||||
else if (profile.Education.Count > 0) profile.Education[^1].Details.Add(line);
|
||||
else Unknown(line);
|
||||
break;
|
||||
case "certifications": profile.Certifications.Add(new() { Name = parts[0], Issuer = parts.ElementAtOrDefault(1), Date = parts.ElementAtOrDefault(2) }); break;
|
||||
case "projects":
|
||||
if (profile.Projects.Count == 0 || parts.Length > 1 || block.Kind == "title") profile.Projects.Add(new() { Name = parts[0], Role = parts.ElementAtOrDefault(1) });
|
||||
else profile.Projects[^1].Bullets.Add(line);
|
||||
break;
|
||||
case "interests": profile.Interests.Add(line); break;
|
||||
case "awards": profile.Awards.Add(line); break;
|
||||
case "publications": profile.Publications.Add(line); break;
|
||||
case "organisations": profile.Organisations.Add(line); break;
|
||||
case "references": profile.References.Add(line); break;
|
||||
default: Unknown(line); break;
|
||||
}
|
||||
}
|
||||
if (unassigned.Count > 0) profile.OtherSections.Add(new() { Title = "Unclassified source", Items = unassigned });
|
||||
return profile;
|
||||
}
|
||||
|
||||
private static (string? Start, string? End) Dates(string? text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text)) return (null, null);
|
||||
var dates = Regex.Split(text, @"\s+(?:-|–|—|to|til)\s+|(?<=\d{4})[–—](?=\d{4})", RegexOptions.IgnoreCase);
|
||||
return dates.Length == 2 ? (dates[0].Trim(), dates[1].Trim()) : (text.Trim(), null);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,241 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Nodes;
|
||||
using System.Text.RegularExpressions;
|
||||
using JobTrackerApi.Models;
|
||||
|
||||
namespace JobTrackerApi.Services;
|
||||
|
||||
public sealed record CvImportBlock(string Id, string Text, string Kind, int? Page);
|
||||
public sealed record CvImportDocument(string Text, bool OcrUsed, string? ContentType, int? PageCount, int Characters, string? FileName, IReadOnlyList<CvImportBlock> Blocks, IReadOnlyList<string> Warnings, string Method);
|
||||
public sealed record CvImportDecision(string Id, string Action, string? Value = null);
|
||||
public sealed record CvImportField(string Id, string Label, string? CurrentValue, string ExtractedValue, string? SourceSnippet, int? SourcePage, string Method, string ReviewState);
|
||||
public sealed record CvImportGroup(string Id, string Category, string Label, string? MatchDescription, IReadOnlyList<CvImportField> Fields);
|
||||
public sealed record CvImportReview(int RunId, string Status, int Revision, string? FileName, string SourceUrl, string? SourceText, IReadOnlyList<string> Warnings, IReadOnlyList<CvImportGroup> Groups);
|
||||
|
||||
// Paths are generated from the canonical typed schema, never accepted as arbitrary JSON patches.
|
||||
public static class CvDocumentReview
|
||||
{
|
||||
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web);
|
||||
private static readonly HashSet<string> Excluded = new(StringComparer.Ordinal) { "id", "metadata", "version", "sections", "startDate", "endDate", "dateNormalized" };
|
||||
private static JsonObject Node(StructuredCvProfile profile) => JsonSerializer.SerializeToNode(profile, Json)!.AsObject();
|
||||
private static string Clean(string text) => Regex.Replace(text, @"\s+", " ").Trim();
|
||||
|
||||
public static StructuredCvProfile Ground(StructuredCvProfile candidate, CvImportDocument document)
|
||||
{
|
||||
var node = Node(candidate);
|
||||
node["sections"] = new JsonArray();
|
||||
var metadata = new Dictionary<string, StructuredCvFieldMetadata>();
|
||||
void Walk(JsonNode? value, string path)
|
||||
{
|
||||
if (value is JsonObject obj)
|
||||
{
|
||||
foreach (var entry in obj.ToList())
|
||||
{
|
||||
if (Excluded.Contains(entry.Key)) { if (entry.Key is "startDate" or "endDate" or "dateNormalized" or "id") obj[entry.Key] = null; continue; }
|
||||
Walk(entry.Value, path.Length == 0 ? entry.Key : path + "." + entry.Key);
|
||||
}
|
||||
}
|
||||
else if (value is JsonArray array)
|
||||
{
|
||||
for (var i = array.Count - 1; i >= 0; i--)
|
||||
{
|
||||
if (array[i] is JsonValue scalar && scalar.TryGetValue<string>(out var text) && !document.Blocks.Any(b => Clean(b.Text).Contains(Clean(text), StringComparison.OrdinalIgnoreCase))) array.RemoveAt(i);
|
||||
}
|
||||
for (var i = 0; i < array.Count; i++) Walk(array[i], path + "." + i);
|
||||
}
|
||||
else if (value is JsonValue scalar)
|
||||
{
|
||||
if (scalar.TryGetValue<bool>(out var flag))
|
||||
{
|
||||
if (flag && value.Parent is JsonObject record)
|
||||
{
|
||||
var end = record["end"]?.ToString();
|
||||
var marker = end is not null && Regex.IsMatch(end, @"^(present|current|now|nå|dags dato|pågående)$", RegexOptions.IgnoreCase);
|
||||
var evidence = marker ? document.Blocks.FirstOrDefault(b => Clean(b.Text).Contains(Clean(end!), StringComparison.OrdinalIgnoreCase)) : null;
|
||||
if (evidence is null) record[path.Split('.').Last()] = false;
|
||||
else metadata[path] = new() { Method = "normalized-current-marker", SourceBlockId = evidence.Id, SourcePage = evidence.Page, SourceSnippet = end, ReviewState = "suggested" };
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!scalar.TryGetValue<string>(out var text) || string.IsNullOrWhiteSpace(text)) return;
|
||||
var block = document.Blocks.FirstOrDefault(b => Clean(b.Text).Contains(Clean(text), StringComparison.OrdinalIgnoreCase));
|
||||
if (block is null) { if (value.Parent is JsonObject parent) parent[path.Split('.').Last()] = null; return; }
|
||||
metadata[path] = new() { Method = "deterministic-source", SourceBlockId = block.Id, SourcePage = block.Page, SourceSnippet = text.Length > 500 ? text[..500] : text, ReviewState = "suggested" };
|
||||
}
|
||||
}
|
||||
Walk(node, "");
|
||||
var result = node.Deserialize<StructuredCvProfile>(Json)!;
|
||||
result.Metadata = new() { Fields = metadata };
|
||||
return result;
|
||||
}
|
||||
|
||||
private sealed record GroupMap(CvImportGroup Review, JsonNode Incoming, string Category, int? ExistingIndex, bool IsArray);
|
||||
public static IReadOnlyList<CvImportGroup> Build(StructuredCvProfile current, StructuredCvProfile incoming) => Maps(current, incoming).Select(m => m.Review).ToList();
|
||||
|
||||
private static List<GroupMap> Maps(StructuredCvProfile current, StructuredCvProfile incoming)
|
||||
{
|
||||
var old = Node(current);
|
||||
var next = Node(incoming);
|
||||
var groups = new List<GroupMap>();
|
||||
foreach (var category in next)
|
||||
{
|
||||
if (Excluded.Contains(category.Key) || category.Value is null) continue;
|
||||
if (category.Value is JsonArray items)
|
||||
{
|
||||
var existing = old[category.Key]!.AsArray();
|
||||
for (var i = 0; i < items.Count; i++)
|
||||
{
|
||||
var item = items[i];
|
||||
if (item is null) continue;
|
||||
var matches = Enumerable.Range(0, existing.Count).Where(j => Same(category.Key, existing[j], item)).ToList();
|
||||
int? match = matches.Count == 1 ? matches[0] : null;
|
||||
Add(category.Key + "." + i, category.Key, item, match is int index ? existing[index] : null, match, true);
|
||||
}
|
||||
}
|
||||
else Add(category.Key, category.Key, category.Value, old[category.Key], null, false);
|
||||
}
|
||||
return groups;
|
||||
|
||||
void Add(string id, string category, JsonNode item, JsonNode? existing, int? index, bool array)
|
||||
{
|
||||
var fields = new List<CvImportField>();
|
||||
void Walk(JsonNode? value, JsonNode? previous, string path)
|
||||
{
|
||||
if (value is JsonObject obj)
|
||||
{
|
||||
foreach (var property in obj) if (!Excluded.Contains(property.Key)) Walk(property.Value, (previous as JsonObject)?[property.Key], path + "." + property.Key);
|
||||
}
|
||||
else if (value is JsonArray list)
|
||||
{
|
||||
for (var j = 0; j < list.Count; j++)
|
||||
{
|
||||
var index = path == "contact.links" && previous is JsonArray links && list[j] is JsonObject link ? LinkIndex(links, link) : j;
|
||||
Walk(list[j], previous is JsonArray before && index is int matched && matched < before.Count ? before[matched] : null, path + "." + j);
|
||||
}
|
||||
}
|
||||
else if (value is JsonValue scalar)
|
||||
{
|
||||
var text = scalar.ToString();
|
||||
if (string.IsNullOrWhiteSpace(text) || (scalar.TryGetValue<bool>(out var boolean) && !boolean)) return;
|
||||
incoming.Metadata.Fields.TryGetValue(path, out var source);
|
||||
var label = Regex.Replace(path.Split('.').Last(), "([a-z])([A-Z])", "$1 $2");
|
||||
if (int.TryParse(label, out _)) label = "Item";
|
||||
label = char.ToUpperInvariant(label[0]) + label[1..];
|
||||
fields.Add(new(path, label, previous?.ToString(), text, source?.SourceSnippet, source?.SourcePage, source?.Method ?? "deterministic-source", "suggested"));
|
||||
}
|
||||
}
|
||||
Walk(item, existing, id);
|
||||
if (fields.Count > 0) groups.Add(new(new(id, category, category, array ? index.HasValue ? "Matched existing record" : "New record" : null, fields), item, category, index, array));
|
||||
}
|
||||
}
|
||||
|
||||
private static bool Same(string category, JsonNode? left, JsonNode right)
|
||||
{
|
||||
if (left is JsonValue && right is JsonValue) return string.Equals(left.ToString(), right.ToString(), StringComparison.OrdinalIgnoreCase);
|
||||
var keys = category switch
|
||||
{
|
||||
"jobs" => new[] { "company", "title", "start", "end" },
|
||||
"education" => new[] { "institution", "qualification", "start", "end" },
|
||||
"certifications" => new[] { "name", "issuer", "date" },
|
||||
"projects" => new[] { "name", "role", "start", "end" },
|
||||
"languages" => new[] { "name" },
|
||||
"otherSections" => new[] { "title" },
|
||||
_ => Array.Empty<string>()
|
||||
};
|
||||
return keys.Length > 0 && keys.All(k => !string.IsNullOrWhiteSpace(right[k]?.ToString()) && string.Equals(Clean(left?[k]?.ToString() ?? ""), Clean(right[k]!.ToString()), StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static int? LinkIndex(JsonArray links, JsonObject incoming)
|
||||
{
|
||||
var label = Clean(incoming["label"]?.ToString() ?? "");
|
||||
var matches = Enumerable.Range(0, links.Count).Where(i => label.Length > 0 && string.Equals(Clean(links[i]?["label"]?.ToString() ?? ""), label, StringComparison.OrdinalIgnoreCase)).ToList();
|
||||
return matches.Count == 1 ? matches[0] : null;
|
||||
}
|
||||
|
||||
public static StructuredCvProfile Apply(StructuredCvProfile current, StructuredCvProfile incoming, IReadOnlyList<CvImportDecision> decisions)
|
||||
{
|
||||
var maps = Maps(current, incoming);
|
||||
var known = maps.SelectMany(m => m.Review.Fields).ToDictionary(f => f.Id, StringComparer.Ordinal);
|
||||
if (decisions.Count > 2000 || decisions.Select(d => d.Id).Distinct(StringComparer.Ordinal).Count() != decisions.Count) throw new ArgumentException("Duplicate or excessive review decisions.");
|
||||
foreach (var d in decisions)
|
||||
if (!known.ContainsKey(d.Id) || d.Action is not ("keep" or "use" or "correct" or "add") || (d.Action == "correct" && (string.IsNullOrWhiteSpace(d.Value) || d.Value.Length > 8000 || d.Value.Any(c => char.IsControl(c) && c is not '\n' and not '\r' and not '\t')))) throw new ArgumentException("Invalid review decision.");
|
||||
var selected = decisions.Where(d => d.Action != "keep").ToDictionary(d => d.Id, StringComparer.Ordinal);
|
||||
var result = Node(current);
|
||||
var linkTargets = new Dictionary<int, int>();
|
||||
foreach (var map in maps)
|
||||
{
|
||||
var choices = map.Review.Fields.Where(f => selected.ContainsKey(f.Id)).ToList();
|
||||
if (choices.Count == 0) continue;
|
||||
JsonNode target;
|
||||
string targetPrefix;
|
||||
if (map.IsArray)
|
||||
{
|
||||
var list = result[map.Category]!.AsArray();
|
||||
var index = map.ExistingIndex ?? list.Count;
|
||||
if (!map.ExistingIndex.HasValue) list.Add(map.Incoming is JsonObject ? new JsonObject() : JsonValue.Create(""));
|
||||
target = list[index]!;
|
||||
targetPrefix = map.Category + "." + index;
|
||||
}
|
||||
else { target = result[map.Category]!; targetPrefix = map.Category; }
|
||||
foreach (var field in choices)
|
||||
{
|
||||
var decision = selected[field.Id];
|
||||
var text = decision.Action == "correct" ? decision.Value!.Trim() : field.ExtractedValue;
|
||||
var suffix = field.Id[map.Review.Id.Length..].TrimStart('.');
|
||||
if (map.Category == "contact" && suffix.StartsWith("links.", StringComparison.Ordinal))
|
||||
{
|
||||
var segments = suffix.Split('.');
|
||||
var incomingIndex = int.Parse(segments[1]);
|
||||
if (!linkTargets.TryGetValue(incomingIndex, out var targetIndex))
|
||||
{
|
||||
var links = result["contact"]!["links"]!.AsArray();
|
||||
targetIndex = LinkIndex(links, map.Incoming["links"]![incomingIndex]!.AsObject()) ?? links.Count;
|
||||
linkTargets[incomingIndex] = targetIndex;
|
||||
}
|
||||
suffix = $"links.{targetIndex}.{segments[2]}";
|
||||
}
|
||||
JsonNode value = JsonValue.Create(text)!;
|
||||
if (field.Id.EndsWith(".isCurrent", StringComparison.Ordinal))
|
||||
{
|
||||
if (!bool.TryParse(text, out var boolean)) throw new ArgumentException("isCurrent must be true or false.");
|
||||
value = JsonValue.Create(boolean)!;
|
||||
}
|
||||
if (suffix.Length == 0)
|
||||
{
|
||||
var list = result[map.Category]!.AsArray();
|
||||
list[int.Parse(targetPrefix.Split('.').Last())] = value;
|
||||
}
|
||||
else Set(target, suffix.Split('.'), value);
|
||||
var path = targetPrefix + (suffix.Length == 0 ? "" : "." + suffix);
|
||||
incoming.Metadata.Fields.TryGetValue(field.Id, out var source);
|
||||
var metadataNode = result["metadata"]!["fields"]!.AsObject();
|
||||
metadataNode[path] = JsonSerializer.SerializeToNode(new StructuredCvFieldMetadata { Method = decision.Action == "correct" ? "user-correction" : source?.Method ?? "deterministic-source", SourceSnippet = source?.SourceSnippet, SourcePage = source?.SourcePage, SourceBlockId = source?.SourceBlockId, ReviewState = decision.Action == "correct" ? "corrected" : "accepted", LastUpdatedAtUtc = DateTimeOffset.UtcNow }, Json);
|
||||
}
|
||||
}
|
||||
return result.Deserialize<StructuredCvProfile>(Json)!;
|
||||
}
|
||||
|
||||
private static void Set(JsonNode target, string[] path, JsonNode value)
|
||||
{
|
||||
for (var i = 0; i < path.Length - 1; i++)
|
||||
{
|
||||
var key = path[i];
|
||||
var arrayNext = int.TryParse(path[i + 1], out _);
|
||||
if (target is JsonArray list)
|
||||
{
|
||||
var index = int.Parse(key);
|
||||
while (list.Count <= index) list.Add(null);
|
||||
list[index] ??= arrayNext ? new JsonArray() : new JsonObject();
|
||||
target = list[index]!;
|
||||
}
|
||||
else { target[key] ??= arrayNext ? new JsonArray() : new JsonObject(); target = target[key]!; }
|
||||
}
|
||||
if (target is JsonArray array)
|
||||
{
|
||||
var index = int.Parse(path[^1]);
|
||||
// Individual approved list entries append instead of erasing user-only entries.
|
||||
if (!array.Any(v => v?.ToString() == value.ToString())) array.Add(value);
|
||||
}
|
||||
else target[path[^1]] = value;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
using JobTrackerApi.Data;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace JobTrackerApi.Services;
|
||||
|
||||
public sealed class CvImportRetentionHostedService(IServiceScopeFactory scopes, AppPaths paths, ILogger<CvImportRetentionHostedService> logger) : BackgroundService
|
||||
{
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
using var timer = new PeriodicTimer(TimeSpan.FromHours(1));
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var scope = scopes.CreateAsyncScope();
|
||||
await SweepAsync(scope.ServiceProvider.GetRequiredService<JobTrackerContext>(), paths, DateTimeOffset.UtcNow, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; }
|
||||
catch (Exception) { logger.LogWarning("CV import retention could not complete; it will retry on the next sweep."); }
|
||||
} while (await timer.WaitForNextTickAsync(stoppingToken));
|
||||
}
|
||||
|
||||
public static async Task<int> SweepAsync(JobTrackerContext db, AppPaths paths, DateTimeOffset now, CancellationToken token)
|
||||
{
|
||||
var runs = await db.CvExtractionRuns.IgnoreQueryFilters().Include(r => r.Artifact)
|
||||
.Where(r => r.ParserVersion == "docling-v1" && r.Artifact != null && r.Artifact.StoragePath != "")
|
||||
.OrderBy(r => r.Id).Take(1000).ToListAsync(token);
|
||||
var root = Path.GetFullPath(paths.CvArtifactsRoot) + Path.DirectorySeparatorChar;
|
||||
var removed = 0;
|
||||
foreach (var run in runs)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
if (run.StartedAtUtc > now.AddDays(-7) && (!(run.Status is "cancelled" or "failed" or "discarded") || run.StartedAtUtc > now.AddDays(-1))) continue;
|
||||
var artifact = run.Artifact!;
|
||||
if (artifact.OwnerUserId != run.OwnerUserId) continue;
|
||||
var path = Path.GetFullPath(artifact.StoragePath);
|
||||
if (!path.StartsWith(root, OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal)) continue;
|
||||
// Revoke the operation before deleting its source; a late worker cannot publish a result.
|
||||
var subject = run.Id.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
var operations = await db.UserOperations.IgnoreQueryFilters().Where(o => o.OwnerUserId == run.OwnerUserId && o.SubjectType == CvProcessingQueue.SubjectType && o.SubjectId == subject).ToListAsync(token);
|
||||
foreach (var operation in operations.Where(o => !Models.OperationStatuses.IsTerminal(o.Status)))
|
||||
{
|
||||
operation.Status = Models.OperationStatuses.Cancelled;
|
||||
operation.CancellationRequestedAtUtc = now.UtcDateTime;
|
||||
operation.CompletedAtUtc = now.UtcDateTime;
|
||||
}
|
||||
await db.SaveChangesAsync(token);
|
||||
try
|
||||
{
|
||||
if (File.Exists(path) && (File.GetAttributes(path) & FileAttributes.ReparsePoint) != 0) continue;
|
||||
File.Delete(path);
|
||||
}
|
||||
catch (IOException) { continue; }
|
||||
catch (UnauthorizedAccessException) { continue; }
|
||||
artifact.StoragePath = "";
|
||||
run.DocumentJson = null;
|
||||
run.RawExtractedText = null;
|
||||
run.NormalizedText = null;
|
||||
if (run.Status != "applied")
|
||||
{
|
||||
run.StructuredProfileJson = null;
|
||||
if (run.Status is "pending_review" or "queued" or "running") run.Status = "expired";
|
||||
run.CompletedAtUtc ??= now;
|
||||
run.ErrorMessage = "The source document expired. Upload it again to start a new review.";
|
||||
}
|
||||
removed++;
|
||||
}
|
||||
await db.SaveChangesAsync(token);
|
||||
return removed;
|
||||
}
|
||||
}
|
||||
@@ -64,7 +64,7 @@ public sealed class CvProcessingOperationHandler : IAiOperationHandler
|
||||
try
|
||||
{
|
||||
outcome = await services.GetRequiredService<ProfileCvController>()
|
||||
.ProcessQueuedRunAsync(runId, cancellationToken);
|
||||
.ProcessQueuedRunAsync(runId, cancellationToken, context);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
@@ -109,7 +109,7 @@ public sealed class CvProcessingOperationHandler : IAiOperationHandler
|
||||
}
|
||||
|
||||
return new AiOperationExecutionResult(
|
||||
$"/api/profile-cv/runs/{runId}/diff",
|
||||
$"/api/profile-cv/runs/{runId}/{(await services.GetRequiredService<JobTrackerContext>().CvExtractionRuns.Where(r => r.Id == runId).Select(r => r.ParserVersion).FirstAsync(cancellationToken) == ProfileCvController.DocumentParserVersion ? "review" : "diff")}",
|
||||
outcome.Provider,
|
||||
outcome.Model,
|
||||
outcome.RouteReason ?? "cv_pipeline");
|
||||
|
||||
@@ -39,6 +39,9 @@ public sealed class MeteredSummarizerService(
|
||||
=> MeterAsync("synchronous.generate", instruction + "\n\n" + text, maxLength,
|
||||
ct => inner.GenerateSectionWithMetadataAsync(instruction, text, outputMode, maxLength, minLength, ct), cancellationToken);
|
||||
|
||||
public Task<CvImportDocument?> ExtractCvDocumentAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default)
|
||||
=> inner.ExtractCvDocumentAsync(stream, fileName, contentType, cancellationToken);
|
||||
|
||||
public Task<AiTextExtractionResult?> ExtractTextAsync(
|
||||
Stream stream,
|
||||
string fileName,
|
||||
|
||||
@@ -94,6 +94,8 @@ namespace JobTrackerApi.Services
|
||||
|
||||
public interface ISummarizerService : IAiService
|
||||
{
|
||||
Task<CvImportDocument?> ExtractCvDocumentAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult<CvImportDocument?>(null);
|
||||
new Task<string?> SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40);
|
||||
|
||||
// Older providers cannot approve factual changes without implementing this review boundary.
|
||||
@@ -492,6 +494,32 @@ namespace JobTrackerApi.Services
|
||||
}
|
||||
}
|
||||
|
||||
public async Task<CvImportDocument?> ExtractCvDocumentAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
using var form = new MultipartFormDataContent();
|
||||
using var file = new StreamContent(stream);
|
||||
file.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType ?? "application/octet-stream");
|
||||
form.Add(file, "file", Path.GetFileName(fileName));
|
||||
using var response = await _httpFactory.CreateClient("ai-service").PostAsync("/cv-import/extract", form, cancellationToken);
|
||||
if (!response.IsSuccessStatusCode) throw new AiGenerationException("cv_extraction_failed", "Document extraction is unavailable or the document could not be parsed.", (int)response.StatusCode >= 500);
|
||||
await using var body = await response.Content.ReadAsStreamAsync(cancellationToken);
|
||||
using var bounded = new MemoryStream();
|
||||
var buffer = new byte[8192];
|
||||
int read;
|
||||
while ((read = await body.ReadAsync(buffer, cancellationToken)) > 0)
|
||||
{
|
||||
if (bounded.Length + read > 2 * 1024 * 1024) throw new InvalidOperationException("Document extraction response exceeded its limit.");
|
||||
await bounded.WriteAsync(buffer.AsMemory(0, read), cancellationToken);
|
||||
}
|
||||
using var json = JsonDocument.Parse(bounded.ToArray());
|
||||
var root = json.RootElement;
|
||||
var text = root.GetProperty("text").GetString() ?? "";
|
||||
var blocks = root.GetProperty("blocks").EnumerateArray().Select(b => new CvImportBlock(b.GetProperty("id").GetString() ?? "", b.GetProperty("text").GetString() ?? "", b.GetProperty("kind").GetString() ?? "text", b.TryGetProperty("page", out var page) && page.ValueKind == JsonValueKind.Number ? page.GetInt32() : null)).ToList();
|
||||
var warnings = root.GetProperty("warnings").EnumerateArray().Select(w => w.GetString() ?? "").ToList();
|
||||
if (string.IsNullOrWhiteSpace(text) || text.Length > 500000 || blocks.Count is 0 or > 10000 || blocks.Any(b => b.Id.Length is 0 or > 128 || b.Text.Length > 100000 || b.Page is < 1 or > 1000) || blocks.Select(b => b.Id).Distinct().Count() != blocks.Count || warnings.Count > 100 || warnings.Any(w => w.Length > 1000)) throw new InvalidOperationException("Document extraction returned invalid evidence.");
|
||||
return new CvImportDocument(text, root.GetProperty("ocr_used").GetBoolean(), root.GetProperty("content_type").GetString(), root.TryGetProperty("page_count", out var pages) && pages.ValueKind == JsonValueKind.Number ? pages.GetInt32() : null, text.Length, Path.GetFileName(fileName), blocks, warnings, root.GetProperty("method").GetString() ?? "docling");
|
||||
}
|
||||
|
||||
public async Task<AiTextExtractionResult?> ExtractTextAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(stream);
|
||||
|
||||
@@ -169,8 +169,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
|
||||
.ExecuteUpdateAsync(setters => setters
|
||||
.SetProperty(operation => operation.LeaseExpiresAtUtc, now.Add(leaseDuration))
|
||||
.SetProperty(operation => operation.LastHeartbeatAtUtc, now)
|
||||
.SetProperty(operation => operation.ProgressStage, progressStage)
|
||||
.SetProperty(operation => operation.ProgressPercent, progressPercent),
|
||||
.SetProperty(operation => operation.ProgressStage, operation => progressStage ?? operation.ProgressStage)
|
||||
.SetProperty(operation => operation.ProgressPercent, operation => progressPercent ?? operation.ProgressPercent),
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user