cesnimda
300 lines
19 KiB
C#
300 lines
19 KiB
C#
using System.Text.Json;
|
|
using System.Text.Json.Nodes;
|
|
using System.Text.RegularExpressions;
|
|
using JobTrackerApi.Models;
|
|
|
|
namespace JobTrackerApi.Services;
|
|
|
|
public sealed record CvImportBlock(string Id, string Text, string Kind, int? Page);
|
|
public sealed record CvImportDocument(string Text, bool OcrUsed, string? ContentType, int? PageCount, int Characters, string? FileName, IReadOnlyList<CvImportBlock> Blocks, IReadOnlyList<string> Warnings, string Method);
|
|
public sealed record CvImportDecision(string Id, string Action, string? Value = null);
|
|
public sealed record CvImportField(string Id, string Label, string? CurrentValue, string ExtractedValue, string? SourceSnippet, int? SourcePage, string Method, string ReviewState);
|
|
public sealed record CvImportGroup(string Id, string Category, string Label, string? MatchDescription, IReadOnlyList<CvImportField> Fields);
|
|
public sealed record CvImportReview(int RunId, string Status, int Revision, string? FileName, string SourceUrl, string? SourceText, IReadOnlyList<string> Warnings, IReadOnlyList<CvImportGroup> Groups, IReadOnlyList<CvImportBlock> SourceBlocks = null!, IReadOnlyList<CvImportMappingCategory> MappingCategories = null!, IReadOnlyList<CvImportMappingTarget> Targets = null!);
|
|
|
|
// Paths are generated from the canonical typed schema, never accepted as arbitrary JSON patches.
|
|
public static class CvDocumentReview
|
|
{
|
|
private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web);
|
|
private static readonly HashSet<string> Excluded = new(StringComparer.Ordinal) { "id", "metadata", "version", "sections", "startDate", "endDate", "dateNormalized" };
|
|
private static JsonObject Node(StructuredCvProfile profile) => JsonSerializer.SerializeToNode(profile, Json)!.AsObject();
|
|
private static string Clean(string text) => Regex.Replace(text, @"\s+", " ").Trim();
|
|
|
|
public static StructuredCvProfile Ground(StructuredCvProfile candidate, CvImportDocument document)
|
|
{
|
|
var original = Node(candidate);
|
|
var supplied = candidate.Metadata.Fields.ToDictionary(f => f.Key, f => f.Value, StringComparer.Ordinal);
|
|
var node = Node(StructuredCvProfileJson.NormalizeForPersistence(candidate));
|
|
node["sections"] = new JsonArray();
|
|
var selected = new Dictionary<string, StructuredCvFieldMetadata>(StringComparer.Ordinal);
|
|
Reindex(original, node, "", "");
|
|
var metadata = new Dictionary<string, StructuredCvFieldMetadata>();
|
|
|
|
// Persistence trims/deduplicates lists; carry selected evidence to the retained leaf's index.
|
|
void Reindex(JsonNode? before, JsonNode? after, string oldPath, string newPath)
|
|
{
|
|
if (after is JsonObject obj)
|
|
{
|
|
foreach (var entry in obj.Where(e => !Excluded.Contains(e.Key))) Reindex((before as JsonObject)?[entry.Key], entry.Value, oldPath.Length == 0 ? entry.Key : oldPath + "." + entry.Key, newPath.Length == 0 ? entry.Key : newPath + "." + entry.Key);
|
|
}
|
|
else if (after is JsonArray list && before is JsonArray old)
|
|
{
|
|
var used = new HashSet<int>();
|
|
for (var i = 0; i < list.Count; i++)
|
|
{
|
|
var index = Enumerable.Range(0, old.Count).FirstOrDefault(j => !used.Contains(j) && Equivalent(old[j], list[i]), -1);
|
|
if (index < 0) continue;
|
|
used.Add(index);
|
|
Reindex(old[index], list[i], oldPath + "." + index, newPath + "." + i);
|
|
}
|
|
}
|
|
else if (supplied.TryGetValue(oldPath, out var evidence)) selected[newPath] = evidence;
|
|
}
|
|
bool Equivalent(JsonNode? before, JsonNode? after)
|
|
{
|
|
if (after is JsonObject obj) return obj.Where(e => !Excluded.Contains(e.Key)).All(e => Equivalent((before as JsonObject)?[e.Key], e.Value));
|
|
if (after is JsonArray array) return before is JsonArray old && array.All(item => old.Any(prior => Equivalent(prior, item)));
|
|
return string.Equals(Clean(before?.ToString() ?? ""), Clean(after?.ToString() ?? ""), StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
CvImportBlock? Evidence(string path, string value)
|
|
{
|
|
if (!selected.TryGetValue(path, out var evidence)) return document.Blocks.FirstOrDefault(b => Clean(b.Text).Contains(Clean(value), StringComparison.OrdinalIgnoreCase));
|
|
var block = document.Blocks.FirstOrDefault(b => b.Id == evidence.SourceBlockId);
|
|
return block is not null && !string.IsNullOrWhiteSpace(evidence.SourceSnippet)
|
|
&& Clean(block.Text).Contains(Clean(value), StringComparison.OrdinalIgnoreCase)
|
|
&& Clean(block.Text).Contains(Clean(evidence.SourceSnippet), StringComparison.OrdinalIgnoreCase)
|
|
&& Clean(evidence.SourceSnippet).Contains(Clean(value), StringComparison.OrdinalIgnoreCase) ? block : null;
|
|
}
|
|
void Walk(JsonNode? value, string path)
|
|
{
|
|
if (value is JsonObject obj)
|
|
{
|
|
foreach (var entry in obj.ToList())
|
|
{
|
|
if (Excluded.Contains(entry.Key)) { if (entry.Key is "startDate" or "endDate" or "dateNormalized" or "id") obj[entry.Key] = null; continue; }
|
|
Walk(entry.Value, path.Length == 0 ? entry.Key : path + "." + entry.Key);
|
|
}
|
|
}
|
|
else if (value is JsonArray array)
|
|
{
|
|
var retained = new List<StructuredCvFieldMetadata?>();
|
|
for (var i = 0; i < array.Count; i++) { selected.TryGetValue(path + "." + i, out var evidence); retained.Add(evidence); }
|
|
for (var i = array.Count - 1; i >= 0; i--)
|
|
{
|
|
if (array[i] is JsonValue scalar && scalar.TryGetValue<string>(out var text) && Evidence(path + "." + i, text) is null) { array.RemoveAt(i); retained.RemoveAt(i); }
|
|
}
|
|
for (var i = 0; i < retained.Count; i++)
|
|
if (array[i] is JsonValue) { selected.Remove(path + "." + i); if (retained[i] is { } evidence) selected[path + "." + i] = evidence; }
|
|
for (var i = 0; i < array.Count; i++) Walk(array[i], path + "." + i);
|
|
}
|
|
else if (value is JsonValue scalar)
|
|
{
|
|
if (scalar.TryGetValue<bool>(out var flag))
|
|
{
|
|
if (flag && value.Parent is JsonObject record)
|
|
{
|
|
var end = record["end"]?.ToString();
|
|
var marker = end is not null && Regex.IsMatch(end, @"^(present|current|now|nå|dags dato|pågående)$", RegexOptions.IgnoreCase);
|
|
var evidence = marker ? Evidence(path, end!) : null;
|
|
if (evidence is null) record[path.Split('.').Last()] = false;
|
|
else metadata[path] = new() { Method = "normalized-current-marker", SourceBlockId = evidence.Id, SourcePage = evidence.Page, SourceSnippet = end, ReviewState = "suggested" };
|
|
}
|
|
return;
|
|
}
|
|
if (!scalar.TryGetValue<string>(out var text) || string.IsNullOrWhiteSpace(text)) return;
|
|
var block = Evidence(path, text);
|
|
if (block is null) { if (value.Parent is JsonObject parent) parent[path.Split('.').Last()] = null; return; }
|
|
var groupingNeedsReview = selected.TryGetValue(path, out var selectedEvidence) && selectedEvidence.Method == "legacy-paragraph-continuation";
|
|
metadata[path] = new() { Method = groupingNeedsReview ? "legacy-paragraph-continuation" : "deterministic-source", SourceBlockId = block.Id, SourcePage = block.Page, SourceSnippet = text.Length > 500 ? text[..500] : text, ReviewState = groupingNeedsReview ? "needs_review" : "suggested" };
|
|
}
|
|
}
|
|
Walk(node, "");
|
|
var result = node.Deserialize<StructuredCvProfile>(Json)!;
|
|
result.Metadata = new() { Fields = metadata };
|
|
return result;
|
|
}
|
|
|
|
private sealed record GroupMap(CvImportGroup Review, JsonNode Incoming, string Category, int? ExistingIndex, bool IsArray);
|
|
public static IReadOnlyList<CvImportGroup> Build(StructuredCvProfile current, StructuredCvProfile incoming) => Maps(current, incoming).Select(m => m.Review).ToList();
|
|
|
|
private static List<GroupMap> Maps(StructuredCvProfile current, StructuredCvProfile incoming)
|
|
{
|
|
var old = Node(current);
|
|
var next = Node(incoming);
|
|
var groups = new List<GroupMap>();
|
|
foreach (var category in next)
|
|
{
|
|
if (Excluded.Contains(category.Key) || category.Value is null) continue;
|
|
if (category.Value is JsonArray items)
|
|
{
|
|
var existing = old[category.Key]!.AsArray();
|
|
for (var i = 0; i < items.Count; i++)
|
|
{
|
|
var item = items[i];
|
|
if (item is null) continue;
|
|
var matches = Enumerable.Range(0, existing.Count).Where(j => Same(category.Key, existing[j], item)).ToList();
|
|
int? match = matches.Count == 1 ? matches[0] : null;
|
|
Add(category.Key + "." + i, category.Key, item, match is int index ? existing[index] : null, match, true);
|
|
}
|
|
}
|
|
else Add(category.Key, category.Key, category.Value, old[category.Key], null, false);
|
|
}
|
|
return groups;
|
|
|
|
void Add(string id, string category, JsonNode item, JsonNode? existing, int? index, bool array)
|
|
{
|
|
var fields = new List<CvImportField>();
|
|
void Walk(JsonNode? value, JsonNode? previous, string path)
|
|
{
|
|
if (value is JsonObject obj)
|
|
{
|
|
foreach (var property in obj) if (!Excluded.Contains(property.Key)) Walk(property.Value, (previous as JsonObject)?[property.Key], path + "." + property.Key);
|
|
}
|
|
else if (value is JsonArray list)
|
|
{
|
|
for (var j = 0; j < list.Count; j++)
|
|
{
|
|
var index = path == "contact.links" && previous is JsonArray links && list[j] is JsonObject link ? LinkIndex(links, link) : j;
|
|
Walk(list[j], previous is JsonArray before && index is int matched && matched < before.Count ? before[matched] : null, path + "." + j);
|
|
}
|
|
}
|
|
else if (value is JsonValue scalar)
|
|
{
|
|
var text = scalar.ToString();
|
|
if (string.IsNullOrWhiteSpace(text) || (scalar.TryGetValue<bool>(out var boolean) && !boolean)) return;
|
|
incoming.Metadata.Fields.TryGetValue(path, out var source);
|
|
var label = Regex.Replace(path.Split('.').Last(), "([a-z])([A-Z])", "$1 $2");
|
|
if (int.TryParse(label, out _)) label = "Item";
|
|
label = char.ToUpperInvariant(label[0]) + label[1..];
|
|
fields.Add(new(path, label, previous?.ToString(), text, source?.SourceSnippet, source?.SourcePage, source?.Method ?? "deterministic-source", source?.ReviewState ?? "suggested"));
|
|
}
|
|
}
|
|
Walk(item, existing, id);
|
|
if (fields.Count > 0) groups.Add(new(new(id, category, category, array ? index.HasValue ? "Matched existing record" : "New record" : null, fields), item, category, index, array));
|
|
}
|
|
}
|
|
|
|
private static bool Same(string category, JsonNode? left, JsonNode right)
|
|
{
|
|
if (left is JsonValue && right is JsonValue) return string.Equals(left.ToString(), right.ToString(), StringComparison.OrdinalIgnoreCase);
|
|
var keys = category switch
|
|
{
|
|
"jobs" => new[] { "company", "title", "start", "end" },
|
|
"education" => new[] { "institution", "qualification", "start", "end" },
|
|
"certifications" => new[] { "name", "issuer", "date" },
|
|
"projects" => new[] { "name", "role", "start", "end" },
|
|
"languages" => new[] { "name" },
|
|
"otherSections" => new[] { "title" },
|
|
_ => Array.Empty<string>()
|
|
};
|
|
return keys.Length > 0 && keys.All(k => !string.IsNullOrWhiteSpace(right[k]?.ToString()) && string.Equals(Clean(left?[k]?.ToString() ?? ""), Clean(right[k]!.ToString()), StringComparison.OrdinalIgnoreCase));
|
|
}
|
|
|
|
private static int? LinkIndex(JsonArray links, JsonObject incoming)
|
|
{
|
|
var label = Clean(incoming["label"]?.ToString() ?? "");
|
|
var matches = Enumerable.Range(0, links.Count).Where(i => label.Length > 0 && string.Equals(Clean(links[i]?["label"]?.ToString() ?? ""), label, StringComparison.OrdinalIgnoreCase)).ToList();
|
|
return matches.Count == 1 ? matches[0] : null;
|
|
}
|
|
|
|
public static StructuredCvProfile Apply(StructuredCvProfile current, StructuredCvProfile incoming, IReadOnlyList<CvImportDecision> decisions)
|
|
{
|
|
var maps = Maps(current, incoming);
|
|
var known = maps.SelectMany(m => m.Review.Fields).ToDictionary(f => f.Id, StringComparer.Ordinal);
|
|
if (decisions.Count > 2000 || decisions.Select(d => d.Id).Distinct(StringComparer.Ordinal).Count() != decisions.Count) throw new ArgumentException("Duplicate or excessive review decisions.");
|
|
foreach (var d in decisions)
|
|
if (!known.ContainsKey(d.Id) || d.Action is not ("keep" or "use" or "correct" or "add") || (d.Action == "correct" && (string.IsNullOrWhiteSpace(d.Value) || d.Value.Length > 8000 || d.Value.Any(c => char.IsControl(c) && c is not '\n' and not '\r' and not '\t')))) throw new ArgumentException("Invalid review decision.");
|
|
var selected = decisions.Where(d => d.Action != "keep").ToDictionary(d => d.Id, StringComparer.Ordinal);
|
|
var result = Node(current);
|
|
var linkTargets = new Dictionary<int, int>();
|
|
var newTargets = new Dictionary<string, int>(StringComparer.Ordinal);
|
|
foreach (var map in maps)
|
|
{
|
|
var choices = map.Review.Fields.Where(f => selected.ContainsKey(f.Id)).ToList();
|
|
if (choices.Count == 0) continue;
|
|
JsonNode target;
|
|
string targetPrefix;
|
|
if (map.IsArray)
|
|
{
|
|
var list = result[map.Category]!.AsArray();
|
|
var identity = map.Category + "|" + map.Incoming.ToJsonString();
|
|
var scalarText = selected[choices[0].Id].Action == "correct" ? selected[choices[0].Id].Value! : choices[0].ExtractedValue;
|
|
var matches = Enumerable.Range(0, list.Count).Where(i => map.Incoming is JsonValue
|
|
? string.Equals(Clean(list[i]?.ToString() ?? ""), Clean(scalarText), StringComparison.OrdinalIgnoreCase)
|
|
: Same(map.Category, list[i], map.Incoming)).ToList();
|
|
var index = map.ExistingIndex ?? (newTargets.TryGetValue(identity, out var prior) ? prior : matches.Count == 1 ? matches[0] : list.Count);
|
|
if (index == list.Count) list.Add(map.Incoming is JsonObject ? new JsonObject() : JsonValue.Create(""));
|
|
newTargets[identity] = index;
|
|
target = list[index]!;
|
|
targetPrefix = map.Category + "." + index;
|
|
}
|
|
else { target = result[map.Category]!; targetPrefix = map.Category; }
|
|
foreach (var field in choices)
|
|
{
|
|
var decision = selected[field.Id];
|
|
var text = decision.Action == "correct" ? decision.Value!.Trim() : field.ExtractedValue;
|
|
var suffix = field.Id[map.Review.Id.Length..].TrimStart('.');
|
|
if (map.Category == "contact" && suffix.StartsWith("links.", StringComparison.Ordinal))
|
|
{
|
|
var segments = suffix.Split('.');
|
|
var incomingIndex = int.Parse(segments[1]);
|
|
if (!linkTargets.TryGetValue(incomingIndex, out var targetIndex))
|
|
{
|
|
var links = result["contact"]!["links"]!.AsArray();
|
|
targetIndex = LinkIndex(links, map.Incoming["links"]![incomingIndex]!.AsObject()) ?? links.Count;
|
|
linkTargets[incomingIndex] = targetIndex;
|
|
}
|
|
suffix = $"links.{targetIndex}.{segments[2]}";
|
|
}
|
|
JsonNode value = JsonValue.Create(text)!;
|
|
if (field.Id.EndsWith(".isCurrent", StringComparison.Ordinal))
|
|
{
|
|
if (!bool.TryParse(text, out var boolean)) throw new ArgumentException("isCurrent must be true or false.");
|
|
value = JsonValue.Create(boolean)!;
|
|
}
|
|
if (suffix.Length == 0)
|
|
{
|
|
var list = result[map.Category]!.AsArray();
|
|
list[int.Parse(targetPrefix.Split('.').Last())] = value;
|
|
}
|
|
else suffix = Set(target, suffix.Split('.'), value);
|
|
var path = targetPrefix + (suffix.Length == 0 ? "" : "." + suffix);
|
|
incoming.Metadata.Fields.TryGetValue(field.Id, out var source);
|
|
var metadataNode = result["metadata"]!["fields"]!.AsObject();
|
|
metadataNode[path] = JsonSerializer.SerializeToNode(new StructuredCvFieldMetadata { Method = decision.Action == "correct" ? "user-correction" : source?.Method ?? "deterministic-source", SourceSnippet = source?.SourceSnippet, SourcePage = source?.SourcePage, SourceBlockId = source?.SourceBlockId, SourceArtifactId = source?.SourceArtifactId, SourceExtractionRunId = source?.SourceExtractionRunId, SourceFileSha256 = source?.SourceFileSha256, ReviewState = decision.Action == "correct" ? "corrected" : "accepted", LastUpdatedAtUtc = DateTimeOffset.UtcNow }, Json);
|
|
}
|
|
}
|
|
var profile = result.Deserialize<StructuredCvProfile>(Json)!;
|
|
var linkCount = profile.Contact.Links.Count;
|
|
profile = StructuredCvProfileJson.NormalizeForPersistence(profile);
|
|
if (profile.Contact.Links.Count != linkCount) throw new ArgumentException("A reviewed contact link is incomplete, duplicated or exceeds the supported limit. Keep the existing link or correct its URL.");
|
|
return profile;
|
|
}
|
|
|
|
private static string Set(JsonNode target, string[] path, JsonNode value)
|
|
{
|
|
for (var i = 0; i < path.Length - 1; i++)
|
|
{
|
|
var key = path[i];
|
|
var arrayNext = int.TryParse(path[i + 1], out _);
|
|
if (target is JsonArray list)
|
|
{
|
|
var index = int.Parse(key);
|
|
while (list.Count <= index) list.Add(null);
|
|
list[index] ??= arrayNext ? new JsonArray() : new JsonObject();
|
|
target = list[index]!;
|
|
}
|
|
else { target[key] ??= arrayNext ? new JsonArray() : new JsonObject(); target = target[key]!; }
|
|
}
|
|
if (target is JsonArray array)
|
|
{
|
|
// Individual approved list entries append instead of erasing user-only entries.
|
|
var index = Enumerable.Range(0, array.Count).FirstOrDefault(i => string.Equals(array[i]?.ToString(), value.ToString(), StringComparison.OrdinalIgnoreCase), -1);
|
|
if (index < 0) { index = array.Count; array.Add(value); }
|
|
path[^1] = index.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
|
}
|
|
else target[path[^1]] = value;
|
|
return string.Join('.', path);
|
|
}
|
|
}
|