using System.Text.Json; using System.Text.Json.Nodes; using System.Text.RegularExpressions; using JobTrackerApi.Models; namespace JobTrackerApi.Services; public sealed record CvImportBlock(string Id, string Text, string Kind, int? Page); public sealed record CvImportDocument(string Text, bool OcrUsed, string? ContentType, int? PageCount, int Characters, string? FileName, IReadOnlyList Blocks, IReadOnlyList Warnings, string Method); public sealed record CvImportDecision(string Id, string Action, string? Value = null); public sealed record CvImportField(string Id, string Label, string? CurrentValue, string ExtractedValue, string? SourceSnippet, int? SourcePage, string Method, string ReviewState); public sealed record CvImportGroup(string Id, string Category, string Label, string? MatchDescription, IReadOnlyList Fields); public sealed record CvImportReview(int RunId, string Status, int Revision, string? FileName, string SourceUrl, string? SourceText, IReadOnlyList Warnings, IReadOnlyList Groups, IReadOnlyList SourceBlocks = null!, IReadOnlyList MappingCategories = null!, IReadOnlyList Targets = null!); // Paths are generated from the canonical typed schema, never accepted as arbitrary JSON patches. public static class CvDocumentReview { private static readonly JsonSerializerOptions Json = new(JsonSerializerDefaults.Web); private static readonly HashSet Excluded = new(StringComparer.Ordinal) { "id", "metadata", "version", "sections", "startDate", "endDate", "dateNormalized" }; private static JsonObject Node(StructuredCvProfile profile) => JsonSerializer.SerializeToNode(profile, Json)!.AsObject(); private static string Clean(string text) => Regex.Replace(text, @"\s+", " ").Trim(); public static StructuredCvProfile Ground(StructuredCvProfile candidate, CvImportDocument document) { var original = Node(candidate); var supplied = candidate.Metadata.Fields.ToDictionary(f => f.Key, f => f.Value, StringComparer.Ordinal); var node = Node(StructuredCvProfileJson.NormalizeForPersistence(candidate)); node["sections"] = new JsonArray(); var selected = new Dictionary(StringComparer.Ordinal); Reindex(original, node, "", ""); var metadata = new Dictionary(); // Persistence trims/deduplicates lists; carry selected evidence to the retained leaf's index. void Reindex(JsonNode? before, JsonNode? after, string oldPath, string newPath) { if (after is JsonObject obj) { foreach (var entry in obj.Where(e => !Excluded.Contains(e.Key))) Reindex((before as JsonObject)?[entry.Key], entry.Value, oldPath.Length == 0 ? entry.Key : oldPath + "." + entry.Key, newPath.Length == 0 ? entry.Key : newPath + "." + entry.Key); } else if (after is JsonArray list && before is JsonArray old) { var used = new HashSet(); for (var i = 0; i < list.Count; i++) { var index = Enumerable.Range(0, old.Count).FirstOrDefault(j => !used.Contains(j) && Equivalent(old[j], list[i]), -1); if (index < 0) continue; used.Add(index); Reindex(old[index], list[i], oldPath + "." + index, newPath + "." + i); } } else if (supplied.TryGetValue(oldPath, out var evidence)) selected[newPath] = evidence; } bool Equivalent(JsonNode? before, JsonNode? after) { if (after is JsonObject obj) return obj.Where(e => !Excluded.Contains(e.Key)).All(e => Equivalent((before as JsonObject)?[e.Key], e.Value)); if (after is JsonArray array) return before is JsonArray old && array.All(item => old.Any(prior => Equivalent(prior, item))); return string.Equals(Clean(before?.ToString() ?? ""), Clean(after?.ToString() ?? ""), StringComparison.OrdinalIgnoreCase); } CvImportBlock? Evidence(string path, string value) { if (!selected.TryGetValue(path, out var evidence)) return document.Blocks.FirstOrDefault(b => Clean(b.Text).Contains(Clean(value), StringComparison.OrdinalIgnoreCase)); var block = document.Blocks.FirstOrDefault(b => b.Id == evidence.SourceBlockId); return block is not null && !string.IsNullOrWhiteSpace(evidence.SourceSnippet) && Clean(block.Text).Contains(Clean(value), StringComparison.OrdinalIgnoreCase) && Clean(block.Text).Contains(Clean(evidence.SourceSnippet), StringComparison.OrdinalIgnoreCase) && Clean(evidence.SourceSnippet).Contains(Clean(value), StringComparison.OrdinalIgnoreCase) ? block : null; } void Walk(JsonNode? value, string path) { if (value is JsonObject obj) { foreach (var entry in obj.ToList()) { if (Excluded.Contains(entry.Key)) { if (entry.Key is "startDate" or "endDate" or "dateNormalized" or "id") obj[entry.Key] = null; continue; } Walk(entry.Value, path.Length == 0 ? entry.Key : path + "." + entry.Key); } } else if (value is JsonArray array) { var retained = new List(); for (var i = 0; i < array.Count; i++) { selected.TryGetValue(path + "." + i, out var evidence); retained.Add(evidence); } for (var i = array.Count - 1; i >= 0; i--) { if (array[i] is JsonValue scalar && scalar.TryGetValue(out var text) && Evidence(path + "." + i, text) is null) { array.RemoveAt(i); retained.RemoveAt(i); } } for (var i = 0; i < retained.Count; i++) if (array[i] is JsonValue) { selected.Remove(path + "." + i); if (retained[i] is { } evidence) selected[path + "." + i] = evidence; } for (var i = 0; i < array.Count; i++) Walk(array[i], path + "." + i); } else if (value is JsonValue scalar) { if (scalar.TryGetValue(out var flag)) { if (flag && value.Parent is JsonObject record) { var end = record["end"]?.ToString(); var marker = end is not null && Regex.IsMatch(end, @"^(present|current|now|nå|dags dato|pågående)$", RegexOptions.IgnoreCase); var evidence = marker ? Evidence(path, end!) : null; if (evidence is null) record[path.Split('.').Last()] = false; else metadata[path] = new() { Method = "normalized-current-marker", SourceBlockId = evidence.Id, SourcePage = evidence.Page, SourceSnippet = end, ReviewState = "suggested" }; } return; } if (!scalar.TryGetValue(out var text) || string.IsNullOrWhiteSpace(text)) return; var block = Evidence(path, text); if (block is null) { if (value.Parent is JsonObject parent) parent[path.Split('.').Last()] = null; return; } var groupingNeedsReview = selected.TryGetValue(path, out var selectedEvidence) && selectedEvidence.Method == "legacy-paragraph-continuation"; metadata[path] = new() { Method = groupingNeedsReview ? "legacy-paragraph-continuation" : "deterministic-source", SourceBlockId = block.Id, SourcePage = block.Page, SourceSnippet = text.Length > 500 ? text[..500] : text, ReviewState = groupingNeedsReview ? "needs_review" : "suggested" }; } } Walk(node, ""); var result = node.Deserialize(Json)!; result.Metadata = new() { Fields = metadata }; return result; } private sealed record GroupMap(CvImportGroup Review, JsonNode Incoming, string Category, int? ExistingIndex, bool IsArray); public static IReadOnlyList Build(StructuredCvProfile current, StructuredCvProfile incoming) => Maps(current, incoming).Select(m => m.Review).ToList(); private static List Maps(StructuredCvProfile current, StructuredCvProfile incoming) { var old = Node(current); var next = Node(incoming); var groups = new List(); foreach (var category in next) { if (Excluded.Contains(category.Key) || category.Value is null) continue; if (category.Value is JsonArray items) { var existing = old[category.Key]!.AsArray(); for (var i = 0; i < items.Count; i++) { var item = items[i]; if (item is null) continue; var matches = Enumerable.Range(0, existing.Count).Where(j => Same(category.Key, existing[j], item)).ToList(); int? match = matches.Count == 1 ? matches[0] : null; Add(category.Key + "." + i, category.Key, item, match is int index ? existing[index] : null, match, true); } } else Add(category.Key, category.Key, category.Value, old[category.Key], null, false); } return groups; void Add(string id, string category, JsonNode item, JsonNode? existing, int? index, bool array) { var fields = new List(); void Walk(JsonNode? value, JsonNode? previous, string path) { if (value is JsonObject obj) { foreach (var property in obj) if (!Excluded.Contains(property.Key)) Walk(property.Value, (previous as JsonObject)?[property.Key], path + "." + property.Key); } else if (value is JsonArray list) { for (var j = 0; j < list.Count; j++) { var index = path == "contact.links" && previous is JsonArray links && list[j] is JsonObject link ? LinkIndex(links, link) : j; Walk(list[j], previous is JsonArray before && index is int matched && matched < before.Count ? before[matched] : null, path + "." + j); } } else if (value is JsonValue scalar) { var text = scalar.ToString(); if (string.IsNullOrWhiteSpace(text) || (scalar.TryGetValue(out var boolean) && !boolean)) return; incoming.Metadata.Fields.TryGetValue(path, out var source); var label = Regex.Replace(path.Split('.').Last(), "([a-z])([A-Z])", "$1 $2"); if (int.TryParse(label, out _)) label = "Item"; label = char.ToUpperInvariant(label[0]) + label[1..]; fields.Add(new(path, label, previous?.ToString(), text, source?.SourceSnippet, source?.SourcePage, source?.Method ?? "deterministic-source", source?.ReviewState ?? "suggested")); } } Walk(item, existing, id); if (fields.Count > 0) groups.Add(new(new(id, category, category, array ? index.HasValue ? "Matched existing record" : "New record" : null, fields), item, category, index, array)); } } private static bool Same(string category, JsonNode? left, JsonNode right) { if (left is JsonValue && right is JsonValue) return string.Equals(left.ToString(), right.ToString(), StringComparison.OrdinalIgnoreCase); var keys = category switch { "jobs" => new[] { "company", "title", "start", "end" }, "education" => new[] { "institution", "qualification", "start", "end" }, "certifications" => new[] { "name", "issuer", "date" }, "projects" => new[] { "name", "role", "start", "end" }, "languages" => new[] { "name" }, "otherSections" => new[] { "title" }, _ => Array.Empty() }; return keys.Length > 0 && keys.All(k => !string.IsNullOrWhiteSpace(right[k]?.ToString()) && string.Equals(Clean(left?[k]?.ToString() ?? ""), Clean(right[k]!.ToString()), StringComparison.OrdinalIgnoreCase)); } private static int? LinkIndex(JsonArray links, JsonObject incoming) { var label = Clean(incoming["label"]?.ToString() ?? ""); var matches = Enumerable.Range(0, links.Count).Where(i => label.Length > 0 && string.Equals(Clean(links[i]?["label"]?.ToString() ?? ""), label, StringComparison.OrdinalIgnoreCase)).ToList(); return matches.Count == 1 ? matches[0] : null; } public static StructuredCvProfile Apply(StructuredCvProfile current, StructuredCvProfile incoming, IReadOnlyList decisions) { var maps = Maps(current, incoming); var known = maps.SelectMany(m => m.Review.Fields).ToDictionary(f => f.Id, StringComparer.Ordinal); if (decisions.Count > 2000 || decisions.Select(d => d.Id).Distinct(StringComparer.Ordinal).Count() != decisions.Count) throw new ArgumentException("Duplicate or excessive review decisions."); foreach (var d in decisions) if (!known.ContainsKey(d.Id) || d.Action is not ("keep" or "use" or "correct" or "add") || (d.Action == "correct" && (string.IsNullOrWhiteSpace(d.Value) || d.Value.Length > 8000 || d.Value.Any(c => char.IsControl(c) && c is not '\n' and not '\r' and not '\t')))) throw new ArgumentException("Invalid review decision."); var selected = decisions.Where(d => d.Action != "keep").ToDictionary(d => d.Id, StringComparer.Ordinal); var result = Node(current); var linkTargets = new Dictionary(); var newTargets = new Dictionary(StringComparer.Ordinal); foreach (var map in maps) { var choices = map.Review.Fields.Where(f => selected.ContainsKey(f.Id)).ToList(); if (choices.Count == 0) continue; JsonNode target; string targetPrefix; if (map.IsArray) { var list = result[map.Category]!.AsArray(); var identity = map.Category + "|" + map.Incoming.ToJsonString(); var scalarText = selected[choices[0].Id].Action == "correct" ? selected[choices[0].Id].Value! : choices[0].ExtractedValue; var matches = Enumerable.Range(0, list.Count).Where(i => map.Incoming is JsonValue ? string.Equals(Clean(list[i]?.ToString() ?? ""), Clean(scalarText), StringComparison.OrdinalIgnoreCase) : Same(map.Category, list[i], map.Incoming)).ToList(); var index = map.ExistingIndex ?? (newTargets.TryGetValue(identity, out var prior) ? prior : matches.Count == 1 ? matches[0] : list.Count); if (index == list.Count) list.Add(map.Incoming is JsonObject ? new JsonObject() : JsonValue.Create("")); newTargets[identity] = index; target = list[index]!; targetPrefix = map.Category + "." + index; } else { target = result[map.Category]!; targetPrefix = map.Category; } foreach (var field in choices) { var decision = selected[field.Id]; var text = decision.Action == "correct" ? decision.Value!.Trim() : field.ExtractedValue; var suffix = field.Id[map.Review.Id.Length..].TrimStart('.'); if (map.Category == "contact" && suffix.StartsWith("links.", StringComparison.Ordinal)) { var segments = suffix.Split('.'); var incomingIndex = int.Parse(segments[1]); if (!linkTargets.TryGetValue(incomingIndex, out var targetIndex)) { var links = result["contact"]!["links"]!.AsArray(); targetIndex = LinkIndex(links, map.Incoming["links"]![incomingIndex]!.AsObject()) ?? links.Count; linkTargets[incomingIndex] = targetIndex; } suffix = $"links.{targetIndex}.{segments[2]}"; } JsonNode value = JsonValue.Create(text)!; if (field.Id.EndsWith(".isCurrent", StringComparison.Ordinal)) { if (!bool.TryParse(text, out var boolean)) throw new ArgumentException("isCurrent must be true or false."); value = JsonValue.Create(boolean)!; } if (suffix.Length == 0) { var list = result[map.Category]!.AsArray(); list[int.Parse(targetPrefix.Split('.').Last())] = value; } else suffix = Set(target, suffix.Split('.'), value); var path = targetPrefix + (suffix.Length == 0 ? "" : "." + suffix); incoming.Metadata.Fields.TryGetValue(field.Id, out var source); var metadataNode = result["metadata"]!["fields"]!.AsObject(); metadataNode[path] = JsonSerializer.SerializeToNode(new StructuredCvFieldMetadata { Method = decision.Action == "correct" ? "user-correction" : source?.Method ?? "deterministic-source", SourceSnippet = source?.SourceSnippet, SourcePage = source?.SourcePage, SourceBlockId = source?.SourceBlockId, SourceArtifactId = source?.SourceArtifactId, SourceExtractionRunId = source?.SourceExtractionRunId, SourceFileSha256 = source?.SourceFileSha256, ReviewState = decision.Action == "correct" ? "corrected" : "accepted", LastUpdatedAtUtc = DateTimeOffset.UtcNow }, Json); } } var profile = result.Deserialize(Json)!; var linkCount = profile.Contact.Links.Count; profile = StructuredCvProfileJson.NormalizeForPersistence(profile); if (profile.Contact.Links.Count != linkCount) throw new ArgumentException("A reviewed contact link is incomplete, duplicated or exceeds the supported limit. Keep the existing link or correct its URL."); return profile; } private static string Set(JsonNode target, string[] path, JsonNode value) { for (var i = 0; i < path.Length - 1; i++) { var key = path[i]; var arrayNext = int.TryParse(path[i + 1], out _); if (target is JsonArray list) { var index = int.Parse(key); while (list.Count <= index) list.Add(null); list[index] ??= arrayNext ? new JsonArray() : new JsonObject(); target = list[index]!; } else { target[key] ??= arrayNext ? new JsonArray() : new JsonObject(); target = target[key]!; } } if (target is JsonArray array) { // Individual approved list entries append instead of erasing user-only entries. var index = Enumerable.Range(0, array.Count).FirstOrDefault(i => string.Equals(array[i]?.ToString(), value.ToString(), StringComparison.OrdinalIgnoreCase), -1); if (index < 0) { index = array.Count; array.Add(value); } path[^1] = index.ToString(System.Globalization.CultureInfo.InvariantCulture); } else target[path[^1]] = value; return string.Join('.', path); } }