fix(cv-qa): preserve ambiguous structures and measure extraction accuracy
This commit is contained in:
cesnimda
@@ -23,28 +23,215 @@ public static class CvDocumentInterpreter
|
||||
{
|
||||
var profile = new StructuredCvProfile();
|
||||
var section = "contact";
|
||||
var sectionTitle = "Contact";
|
||||
var unassigned = new List<string>();
|
||||
void Unknown(string line)
|
||||
string? sectionTitle = null;
|
||||
SourceLine? sectionHeading = null;
|
||||
StructuredCvOtherSection? unknown = null;
|
||||
PendingRecord? pending = null;
|
||||
var projectNamed = false;
|
||||
|
||||
void Evidence(string path, string value, SourceLine source, string method = "deterministic-source", string reviewState = "suggested") =>
|
||||
profile.Metadata.Fields[path] = new() { SourceBlockId = source.Block.Id, SourcePage = source.Block.Page, SourceSnippet = value, Method = method, ReviewState = reviewState };
|
||||
|
||||
void Unknown(SourceLine source)
|
||||
{
|
||||
var other = profile.OtherSections.FirstOrDefault(s => s.Title == sectionTitle);
|
||||
if (other is null) { other = new() { Title = sectionTitle }; profile.OtherSections.Add(other); }
|
||||
other.Items.Add(line);
|
||||
if (unknown is null)
|
||||
{
|
||||
unknown = new() { Title = sectionTitle };
|
||||
profile.OtherSections.Add(unknown);
|
||||
if (sectionHeading is not null) Evidence($"otherSections.{profile.OtherSections.Count - 1}.title", sectionTitle!, sectionHeading);
|
||||
}
|
||||
if (unknown.Items.Contains(source.Text, StringComparer.OrdinalIgnoreCase)) return;
|
||||
Evidence($"otherSections.{profile.OtherSections.Count - 1}.items.{unknown.Items.Count}", source.Text, source);
|
||||
unknown.Items.Add(source.Text);
|
||||
}
|
||||
foreach (var block in blocks)
|
||||
foreach (var source in block.Text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
|
||||
|
||||
void Flush(bool acceptLegacyParagraph = false)
|
||||
{
|
||||
var line = source.Trim().TrimStart('#', '•').Trim();
|
||||
if (line.StartsWith('|')) line = line.Trim('|').Trim();
|
||||
if (Regex.IsMatch(line, @"^[|:\s-]+$")) continue;
|
||||
if (pending is null) return;
|
||||
var paragraph = pending.Paragraph;
|
||||
pending.Paragraph = null;
|
||||
if (acceptLegacyParagraph && paragraph is not null && pending.Resolved)
|
||||
{
|
||||
Set(pending.Category == "education" ? "details" : "bullets", paragraph.Text, paragraph, legacyParagraph: true);
|
||||
paragraph = null;
|
||||
}
|
||||
if (!pending.Resolved)
|
||||
{
|
||||
foreach (var source in pending.Source) Unknown(source);
|
||||
pending = null;
|
||||
return;
|
||||
}
|
||||
string? Value(string key) => pending.Fields.TryGetValue(key, out var field) ? field.Value : null;
|
||||
List<string> List(string key) => pending.Lists.TryGetValue(key, out var list) ? list.Select(f => f.Value).ToList() : new();
|
||||
var index = pending.Category switch { "jobs" => profile.Jobs.Count, "education" => profile.Education.Count, "projects" => profile.Projects.Count, _ => profile.Certifications.Count };
|
||||
var prefix = $"{pending.Category}.{index}";
|
||||
switch (pending.Category)
|
||||
{
|
||||
case "jobs": profile.Jobs.Add(new() { Title = Value("title"), Company = Value("company"), Location = Value("location"), Start = Value("start"), End = Value("end"), IsCurrent = IsCurrent(Value("end")), Bullets = List("bullets"), Skills = List("skills") }); break;
|
||||
case "education": profile.Education.Add(new() { Qualification = Value("qualification"), Institution = Value("institution"), QualificationLevel = Value("qualificationLevel"), Location = Value("location"), Start = Value("start"), End = Value("end"), Details = List("details") }); break;
|
||||
case "projects": profile.Projects.Add(new() { Name = Value("name"), Role = Value("role"), Location = Value("location"), Start = Value("start"), End = Value("end"), Bullets = List("bullets"), Skills = List("skills") }); break;
|
||||
case "certifications": profile.Certifications.Add(new() { Name = Value("name"), Issuer = Value("issuer"), Location = Value("location"), Date = Value("date"), Details = List("details") }); break;
|
||||
}
|
||||
foreach (var field in pending.Fields) Evidence(prefix + "." + field.Key, field.Value.Value, field.Value.Source);
|
||||
foreach (var list in pending.Lists)
|
||||
for (var i = 0; i < list.Value.Count; i++)
|
||||
{
|
||||
var value = list.Value[i];
|
||||
Evidence($"{prefix}.{list.Key}.{i}", value.Value, value.Source,
|
||||
value.LegacyParagraph ? "legacy-paragraph-continuation" : "deterministic-source",
|
||||
value.LegacyParagraph ? "needs_review" : "suggested");
|
||||
}
|
||||
if (pending.Category == "jobs" && IsCurrent(Value("end"))) Evidence(prefix + ".isCurrent", Value("end")!, pending.Fields["end"].Source, "normalized-current-marker");
|
||||
pending = null;
|
||||
unknown = null;
|
||||
if (paragraph is not null) Unknown(paragraph);
|
||||
}
|
||||
|
||||
void Set(string field, string value, SourceLine source, bool legacyParagraph = false)
|
||||
{
|
||||
pending ??= new(section);
|
||||
if (section == "projects" && field == "name") projectNamed = true;
|
||||
if (!pending.Source.Contains(source)) pending.Source.Add(source);
|
||||
if (field is "bullets" or "details" or "skills")
|
||||
{
|
||||
if (!pending.Lists.TryGetValue(field, out var list)) pending.Lists[field] = list = new();
|
||||
var values = field == "skills" ? Regex.Split(value, @"\s*[,;•|]\s*") : new[] { value };
|
||||
foreach (var item in values.Where(v => v.Length > 0))
|
||||
if (!list.Any(v => string.Equals(v.Value, item, StringComparison.OrdinalIgnoreCase))) list.Add(new(item, source, legacyParagraph));
|
||||
}
|
||||
else if (field == "dates")
|
||||
{
|
||||
var (start, end) = Dates(value);
|
||||
if (start is not null) pending.Fields["start"] = new(start, source);
|
||||
if (end is not null) pending.Fields["end"] = new(end, source);
|
||||
}
|
||||
else pending.Fields[field] = new(value, source);
|
||||
}
|
||||
|
||||
void BeginSection(string category, string? title, SourceLine? heading)
|
||||
{
|
||||
Flush(acceptLegacyParagraph: true);
|
||||
section = category;
|
||||
sectionTitle = title;
|
||||
sectionHeading = heading;
|
||||
unknown = null;
|
||||
projectNamed = false;
|
||||
}
|
||||
|
||||
foreach (var block in blocks)
|
||||
{
|
||||
var lines = block.Text.Split('\n', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
string[]? columns = null;
|
||||
var tableHasHeader = block.Kind == "table" && lines.Length > 1 && IsTableSeparator(lines[1]);
|
||||
var unknownTableHeader = false;
|
||||
for (var lineIndex = 0; lineIndex < lines.Length; lineIndex++)
|
||||
{
|
||||
var source = new SourceLine(block, lineIndex, lines[lineIndex].Trim());
|
||||
var line = source.Text.TrimStart('#').Trim();
|
||||
var explicitBullet = Regex.IsMatch(line, @"^(?:[-*•]\s+)");
|
||||
if (explicitBullet) line = Regex.Replace(line, @"^[-*•]\s+", "");
|
||||
if (IsTableSeparator(line)) continue;
|
||||
var parts = line.Trim('|').Split('|', StringSplitOptions.TrimEntries);
|
||||
if (tableHasHeader && lineIndex == 0 && IsRecordSection(section))
|
||||
{
|
||||
columns = parts.Select(p => Field(section, p)).ToArray();
|
||||
var identity = Identity(section);
|
||||
unknownTableHeader = !identity.All(columns.Contains) || columns.Any(c => c.Length == 0) || columns.Distinct().Count() != columns.Length;
|
||||
if (unknownTableHeader) { Flush(); Unknown(source); }
|
||||
continue;
|
||||
}
|
||||
if (unknownTableHeader) { Unknown(source); continue; }
|
||||
if (columns is not null)
|
||||
{
|
||||
Flush(acceptLegacyParagraph: parts.Length == columns.Length);
|
||||
if (parts.Length != columns.Length) { Unknown(source); continue; }
|
||||
for (var i = 0; i < parts.Length; i++) if (parts[i].Length > 0) Set(columns[i], parts[i], source);
|
||||
Flush();
|
||||
continue;
|
||||
}
|
||||
var heading = line.TrimEnd(':');
|
||||
if (Headings.TryGetValue(heading, out var named)) { section = named; sectionTitle = heading; continue; }
|
||||
if (block.Kind == "section_header") { section = "custom"; sectionTitle = heading; continue; }
|
||||
if (!explicitBullet && Headings.TryGetValue(heading, out var named)) { BeginSection(named, heading, source); continue; }
|
||||
var labelled = Regex.Match(line, @"^([^:]{1,40}):\s*(.+)$");
|
||||
var label = labelled.Success ? labelled.Groups[1].Value.Trim().ToLowerInvariant() : "";
|
||||
if (section == "contact" && label is "employer" or "company" or "arbeidsgiver" or "job title" or "stilling") BeginSection("jobs", null, null);
|
||||
if (section == "contact" && label is "institution" or "institusjon" or "qualification") BeginSection("education", null, null);
|
||||
if (IsRecordSection(section) && labelled.Success && Field(section, label) is { Length: > 0 } field)
|
||||
{
|
||||
if (pending?.Paragraph is not null || pending is { Category: "certifications", CertificateInput: not CertificateShape.Explicit }) Flush();
|
||||
// Repeated identity labels begin another occurrence, even when the words are identical.
|
||||
if (pending is not null && ((Identity(section).Contains(field) && (pending.Resolved || pending.Fields.ContainsKey(field)))
|
||||
|| (field == "dates" && (pending.Fields.ContainsKey("start") || pending.Fields.ContainsKey("end")))
|
||||
|| pending.Fields.ContainsKey(field))) Flush();
|
||||
Set(field, labelled.Groups[2].Value.Trim(), source);
|
||||
continue;
|
||||
}
|
||||
if (block.Kind == "section_header" && !explicitBullet) { BeginSection("custom", heading, source); continue; }
|
||||
|
||||
if (IsRecordSection(section))
|
||||
{
|
||||
if (section == "certifications" && (block.Kind == "list_item" || explicitBullet)
|
||||
&& (pending is null || pending.CertificateInput != CertificateShape.Explicit))
|
||||
{
|
||||
Flush();
|
||||
Set("name", line, source);
|
||||
pending!.CertificateInput = CertificateShape.ListItem;
|
||||
continue;
|
||||
}
|
||||
if (pending?.Paragraph is not null && (labelled.Success || IsDateOnly(line) || block.Kind == "list_item" || explicitBullet)) Flush();
|
||||
if ((block.Kind == "list_item" || explicitBullet) && pending?.Resolved == true)
|
||||
{
|
||||
Set(section is "education" or "certifications" ? "details" : "bullets", line, source);
|
||||
continue;
|
||||
}
|
||||
if (!explicitBullet && block.Kind != "list_item" && parts.Length >= 2 && parts.All(p => p.Length > 0))
|
||||
{
|
||||
var keys = section switch { "jobs" => new[] { "title", "company", "dates" }, "education" => new[] { "qualification", "institution", "dates" }, "projects" => new[] { "name", "role", "dates" }, _ => new[] { "name", "issuer", "date" } };
|
||||
Flush(acceptLegacyParagraph: parts.Length <= keys.Length);
|
||||
if (parts.Length > keys.Length) { Unknown(source); continue; }
|
||||
for (var i = 0; i < parts.Length; i++) Set(keys[i], parts[i], source);
|
||||
pending!.LegacyParagraphContext = section is "jobs" or "education" or "projects";
|
||||
continue;
|
||||
}
|
||||
if (IsDateOnly(line) && pending?.Resolved == true && section != "certifications" && !pending.Fields.ContainsKey("start") && !pending.Fields.ContainsKey("end"))
|
||||
{
|
||||
Set("dates", line, source);
|
||||
continue;
|
||||
}
|
||||
// Legacy pipe records allow one deferred paragraph; competing lines remain unresolved.
|
||||
if (pending?.Resolved == true && pending.LegacyParagraphContext && block.Kind == "paragraph" && !explicitBullet && !labelled.Success && !IsDateOnly(line))
|
||||
{
|
||||
if (pending.Paragraph is null) pending.Paragraph = source;
|
||||
else { Flush(); Unknown(source); }
|
||||
continue;
|
||||
}
|
||||
if (section == "certifications" && !explicitBullet && block.Kind == "paragraph" && !labelled.Success
|
||||
&& pending is { CertificateInput: CertificateShape.SingleParagraph or CertificateShape.AmbiguousParagraphs })
|
||||
{
|
||||
// Native PDF table cells can look like names; retain the whole unlabelled cluster.
|
||||
pending.Source.Add(source);
|
||||
pending.CertificateInput = CertificateShape.AmbiguousParagraphs;
|
||||
continue;
|
||||
}
|
||||
Flush();
|
||||
if (section == "projects" && (block.Kind == "title" || !projectNamed))
|
||||
{
|
||||
Set("name", line, source);
|
||||
pending!.LegacyParagraphContext = true;
|
||||
}
|
||||
else if (section == "certifications" && !explicitBullet && block.Kind != "list_item" && !labelled.Success && !IsDateOnly(line))
|
||||
{
|
||||
Set("name", line, source);
|
||||
pending!.CertificateInput = CertificateShape.SingleParagraph;
|
||||
}
|
||||
else Unknown(source);
|
||||
continue;
|
||||
}
|
||||
var contact = Regex.Match(line, @"^(name|navn|full name|email|e-post|phone|telefon|location|sted|headline|title|website|nettside|linkedin|github|portfolio)\s*:\s*(.+)$", RegexOptions.IgnoreCase);
|
||||
if (section == "contact" && contact.Success)
|
||||
{
|
||||
var value = contact.Groups[2].Value.Trim();
|
||||
switch (contact.Groups[1].Value.ToLowerInvariant())
|
||||
var contactField = contact.Groups[1].Value.ToLowerInvariant();
|
||||
switch (contactField)
|
||||
{
|
||||
case "name": case "navn": case "full name": profile.Contact.FullName = value; break;
|
||||
case "email": case "e-post": profile.Contact.Email = value; break;
|
||||
@@ -56,55 +243,100 @@ public static class CvDocumentInterpreter
|
||||
case "github": profile.Contact.GitHub = value; break;
|
||||
case "portfolio": profile.Contact.Links.Add(new() { Label = contact.Groups[1].Value, Url = value }); break;
|
||||
}
|
||||
var key = contactField switch { "name" or "navn" or "full name" => "fullName", "e-post" => "email", "telefon" => "phone", "sted" => "location", "title" => "headline", "nettside" => "website", "linkedin" => "linkedIn", "github" => "gitHub", _ => contactField };
|
||||
if (key != "portfolio") Evidence("contact." + key, value, source);
|
||||
else { Evidence($"contact.links.{profile.Contact.Links.Count - 1}.label", contact.Groups[1].Value, source); Evidence($"contact.links.{profile.Contact.Links.Count - 1}.url", value, source); }
|
||||
continue;
|
||||
}
|
||||
var parts = line.Split('|', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
|
||||
switch (section)
|
||||
{
|
||||
case "contact":
|
||||
var email = Regex.Match(line, @"[\w.+-]+@[\w.-]+\.[\p{L}]{2,}");
|
||||
if (email.Success) profile.Contact.Email ??= email.Value;
|
||||
else if (block.Kind == "title" && profile.Contact.FullName is null) profile.Contact.FullName = line;
|
||||
else unassigned.Add(line);
|
||||
if (email.Success && profile.Contact.Email is null) { profile.Contact.Email = email.Value; Evidence("contact.email", email.Value, source); }
|
||||
else if (block.Kind == "title" && profile.Contact.FullName is null) { profile.Contact.FullName = line; Evidence("contact.fullName", line, source); }
|
||||
else Unknown(source);
|
||||
break;
|
||||
case "summary": profile.Summary.Add(line); break;
|
||||
case "skills": profile.Skills.AddRange(Regex.Split(line, @"\s*[,;•|]\s*").Where(s => s.Length > 0)); break;
|
||||
case "summary": Add(profile.Summary, "summary", line, source); break;
|
||||
case "skills": foreach (var skill in Regex.Split(line, @"\s*[,;•|]\s*").Where(s => s.Length > 0)) Add(profile.Skills, "skills", skill, source); break;
|
||||
case "languages":
|
||||
var language = line.Split(':', 2, StringSplitOptions.TrimEntries);
|
||||
profile.Languages.Add(new() { Name = language[0], Level = language.Length == 2 ? language[1] : null }); break;
|
||||
case "jobs":
|
||||
if (parts.Length >= 2)
|
||||
{
|
||||
var (start, end) = Dates(parts.ElementAtOrDefault(2));
|
||||
profile.Jobs.Add(new() { Title = parts[0], Company = parts[1], Start = start, End = end, IsCurrent = end is not null && Regex.IsMatch(end, @"^(present|current|now|nå|dags dato|pågående)$", RegexOptions.IgnoreCase) });
|
||||
}
|
||||
else if (profile.Jobs.Count > 0) profile.Jobs[^1].Bullets.Add(line.TrimStart('-', ' '));
|
||||
else Unknown(line);
|
||||
var languageIndex = profile.Languages.Count;
|
||||
profile.Languages.Add(new() { Name = language[0], Level = language.Length == 2 ? language[1] : null });
|
||||
Evidence($"languages.{languageIndex}.name", language[0], source);
|
||||
if (language.Length == 2) Evidence($"languages.{languageIndex}.level", language[1], source);
|
||||
break;
|
||||
case "education":
|
||||
if (parts.Length >= 2)
|
||||
{
|
||||
var (start, end) = Dates(parts.ElementAtOrDefault(2));
|
||||
profile.Education.Add(new() { Qualification = parts[0], Institution = parts[1], Start = start, End = end });
|
||||
}
|
||||
else if (profile.Education.Count > 0) profile.Education[^1].Details.Add(line);
|
||||
else Unknown(line);
|
||||
break;
|
||||
case "certifications": profile.Certifications.Add(new() { Name = parts[0], Issuer = parts.ElementAtOrDefault(1), Date = parts.ElementAtOrDefault(2) }); break;
|
||||
case "projects":
|
||||
if (profile.Projects.Count == 0 || parts.Length > 1 || block.Kind == "title") profile.Projects.Add(new() { Name = parts[0], Role = parts.ElementAtOrDefault(1) });
|
||||
else profile.Projects[^1].Bullets.Add(line);
|
||||
break;
|
||||
case "interests": profile.Interests.Add(line); break;
|
||||
case "awards": profile.Awards.Add(line); break;
|
||||
case "publications": profile.Publications.Add(line); break;
|
||||
case "organisations": profile.Organisations.Add(line); break;
|
||||
case "references": profile.References.Add(line); break;
|
||||
default: Unknown(line); break;
|
||||
case "interests": Add(profile.Interests, "interests", line, source); break;
|
||||
case "awards": Add(profile.Awards, "awards", line, source); break;
|
||||
case "publications": Add(profile.Publications, "publications", line, source); break;
|
||||
case "organisations": Add(profile.Organisations, "organisations", line, source); break;
|
||||
case "references": Add(profile.References, "references", line, source); break;
|
||||
default: Unknown(source); break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (unassigned.Count > 0) profile.OtherSections.Add(new() { Title = "Unclassified source", Items = unassigned });
|
||||
Flush(acceptLegacyParagraph: true);
|
||||
return profile;
|
||||
|
||||
void Add(List<string> values, string path, string value, SourceLine source)
|
||||
{
|
||||
if (values.Contains(value, StringComparer.OrdinalIgnoreCase)) return;
|
||||
Evidence($"{path}.{values.Count}", value, source);
|
||||
values.Add(value);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record SourceLine(CvImportBlock Block, int Line, string Text);
|
||||
private sealed record SourceValue(string Value, SourceLine Source, bool LegacyParagraph = false);
|
||||
private enum CertificateShape { Explicit, SingleParagraph, AmbiguousParagraphs, ListItem }
|
||||
private sealed class PendingRecord(string category)
|
||||
{
|
||||
public string Category { get; } = category;
|
||||
public Dictionary<string, SourceValue> Fields { get; } = new();
|
||||
public Dictionary<string, List<SourceValue>> Lists { get; } = new();
|
||||
public List<SourceLine> Source { get; } = new();
|
||||
public bool LegacyParagraphContext { get; set; }
|
||||
public SourceLine? Paragraph { get; set; }
|
||||
public CertificateShape CertificateInput { get; set; }
|
||||
public bool Resolved => CertificateInput != CertificateShape.AmbiguousParagraphs && Identity(Category).All(Fields.ContainsKey);
|
||||
}
|
||||
|
||||
private static bool IsRecordSection(string section) => section is "jobs" or "education" or "projects" or "certifications";
|
||||
private static string[] Identity(string section) => section switch { "jobs" => new[] { "title", "company" }, "education" => new[] { "qualification", "institution" }, _ => new[] { "name" } };
|
||||
private static bool IsTableSeparator(string text) => Regex.IsMatch(text, @"^[|:\s-]+$");
|
||||
private static bool IsCurrent(string? text) => text is not null && Regex.IsMatch(text, @"^(present|current|now|nå|dags dato|pågående)$", RegexOptions.IgnoreCase);
|
||||
|
||||
private static string Field(string section, string label)
|
||||
{
|
||||
label = label.Trim().ToLowerInvariant();
|
||||
if (label is "location" or "sted") return "location";
|
||||
if (label is "dates" or "date range" or "period" or "datoer" or "periode") return section == "certifications" ? "date" : "dates";
|
||||
if (label is "date" or "dato") return section == "certifications" ? "date" : "dates";
|
||||
if (label is "start" or "start date" or "fra") return section == "certifications" ? "" : "start";
|
||||
if (label is "end" or "end date" or "til") return section == "certifications" ? "" : "end";
|
||||
if (label is "description" or "details" or "beskrivelse" or "specialism" or "specialization" or "spesialisering") return section is "education" or "certifications" ? "details" : "bullets";
|
||||
if (section is "jobs" or "projects" && label is "technologies" or "skills" or "teknologier" or "ferdigheter") return "skills";
|
||||
return (section, label) switch
|
||||
{
|
||||
("jobs", "title" or "job title" or "role" or "stilling") => "title",
|
||||
("jobs", "company" or "employer" or "arbeidsgiver") => "company",
|
||||
("education", "qualification" or "degree" or "kvalifikasjon" or "grad") => "qualification",
|
||||
("education", "institution" or "school" or "institusjon" or "lærested") => "institution",
|
||||
("education", "qualification level" or "kvalifikasjonsnivå") => "qualificationLevel",
|
||||
("projects", "project" or "project name" or "name" or "prosjekt") => "name",
|
||||
("projects", "role" or "rolle") => "role",
|
||||
("certifications", "certificate" or "certification" or "name" or "sertifisering") => "name",
|
||||
("certifications", "issuer" or "utsteder") => "issuer",
|
||||
_ => ""
|
||||
};
|
||||
}
|
||||
|
||||
private static bool IsDateOnly(string text)
|
||||
{
|
||||
const string month = @"(?:jan(?:uary|uar)?|feb(?:ruary|ruar)?|mar(?:ch|s)?|apr(?:il)?|may|mai|jun(?:e|i)?|jul(?:y|i)?|aug(?:ust)?|sep(?:tember)?|oct(?:ober)?|okt(?:ober)?|nov(?:ember)?|dec(?:ember)?|des(?:ember)?)";
|
||||
var token = @"(?:\d{4}(?:-\d{2}(?:-\d{2})?)?|\d{1,2}[./]\d{1,2}[./]\d{4}|(?:\d{1,2}\s+)?" + month + @"\.?\s+\d{4})";
|
||||
var (start, end) = Dates(text);
|
||||
return start is not null && Regex.IsMatch(start, "^" + token + "$", RegexOptions.IgnoreCase)
|
||||
&& (end is null || IsCurrent(end) || Regex.IsMatch(end, "^" + token + "$", RegexOptions.IgnoreCase));
|
||||
}
|
||||
|
||||
private static (string? Start, string? End) Dates(string? text)
|
||||
|
||||
@@ -104,7 +104,8 @@ public static class CvDocumentReview
|
||||
if (!scalar.TryGetValue<string>(out var text) || string.IsNullOrWhiteSpace(text)) return;
|
||||
var block = Evidence(path, text);
|
||||
if (block is null) { if (value.Parent is JsonObject parent) parent[path.Split('.').Last()] = null; return; }
|
||||
metadata[path] = new() { Method = "deterministic-source", SourceBlockId = block.Id, SourcePage = block.Page, SourceSnippet = text.Length > 500 ? text[..500] : text, ReviewState = "suggested" };
|
||||
var groupingNeedsReview = selected.TryGetValue(path, out var selectedEvidence) && selectedEvidence.Method == "legacy-paragraph-continuation";
|
||||
metadata[path] = new() { Method = groupingNeedsReview ? "legacy-paragraph-continuation" : "deterministic-source", SourceBlockId = block.Id, SourcePage = block.Page, SourceSnippet = text.Length > 500 ? text[..500] : text, ReviewState = groupingNeedsReview ? "needs_review" : "suggested" };
|
||||
}
|
||||
}
|
||||
Walk(node, "");
|
||||
@@ -165,7 +166,7 @@ public static class CvDocumentReview
|
||||
var label = Regex.Replace(path.Split('.').Last(), "([a-z])([A-Z])", "$1 $2");
|
||||
if (int.TryParse(label, out _)) label = "Item";
|
||||
label = char.ToUpperInvariant(label[0]) + label[1..];
|
||||
fields.Add(new(path, label, previous?.ToString(), text, source?.SourceSnippet, source?.SourcePage, source?.Method ?? "deterministic-source", "suggested"));
|
||||
fields.Add(new(path, label, previous?.ToString(), text, source?.SourceSnippet, source?.SourcePage, source?.Method ?? "deterministic-source", source?.ReviewState ?? "suggested"));
|
||||
}
|
||||
}
|
||||
Walk(item, existing, id);
|
||||
|
||||
Reference in New Issue
Block a user