using System.Text.Json; using JobTrackerApi.Services; using Xunit; namespace JobTrackerApi.Tests; public sealed class CvAiGroundingTests { public static IEnumerable Corpus() { using var document = JsonDocument.Parse(File.ReadAllText(Path.Combine( AppContext.BaseDirectory, "..", "..", "..", "Fixtures", "AiEvaluation", "cv-rewrite-grounding.json"))); foreach (var value in document.RootElement.GetProperty("cases").EnumerateArray()) yield return new object[] { value.Clone() }; } [Theory] [MemberData(nameof(Corpus))] public void Synthetic_semantic_assessments_enforce_the_grounding_contract(JsonElement fixture) { var source = fixture.GetProperty("source").GetString()!; var proposal = fixture.GetProperty("proposal").GetString()!; var verdict = fixture.GetProperty("verdict").GetString()!; // These are controlled evaluator judgments, not live-model factuality measurements. var assessment = JsonSerializer.Serialize(new { verdict, coverageComplete = true, claims = new[] { new { text = proposal, verdict, sourceQuotes = new[] { source } } } }); Assert.Equal(fixture.GetProperty("expectedSupported").GetBoolean(), CvAiGrounding.IsSupported(source, proposal, assessment)); } [Theory] [InlineData("not JSON")] [InlineData("{}")] [InlineData("{\"verdict\":\"supported\",\"coverageComplete\":false,\"claims\":[]}")] [InlineData("{\"verdict\":\"unknown\",\"coverageComplete\":true,\"claims\":[]}")] [InlineData("{\"verdict\":\"SUPPORTED\",\"coverageComplete\":true,\"claims\":[]}")] [InlineData("{\"verdict\":\"supported\",\"coverageComplete\":true,\"claims\":[],\"extra\":true}")] public void Missing_malformed_unknown_or_extra_assessments_fail_closed(string assessment) => Assert.False(CvAiGrounding.IsSupported("Built APIs.", "Developed APIs.", assessment)); [Fact] public void Schema_corruption_fails_even_with_valid_coverage_and_quotes() { var valid = Assessment("Developed APIs.", "supported", "Built APIs."); Assert.True(CvAiGrounding.IsSupported("Built APIs.", "Developed APIs.", valid)); var invalid = new[] { valid.Replace("\"coverageComplete\":true", "\"coverageComplete\":false"), valid.Replace("\"coverageComplete\":true,", ""), valid.Replace("\"coverageComplete\":true", "\"coverageComplete\":\"true\""), valid.Replace("\"supported\"", "\"unknown\""), valid.Insert(valid.Length - 1, ",\"extra\":true"), valid.Insert(valid.Length - 1, ",\"verdict\":\"supported\""), valid.Replace("\"sourceQuotes\":[\"Built APIs.\"]", "\"sourceQuotes\":[]"), valid.Replace("\"sourceQuotes\":[\"Built APIs.\"]", "\"sourceQuotes\":[\"Built APIs.\",\"Job requires SQL.\"]"), valid.Replace("\"text\":\"Developed APIs.\"", "\"text\":\"Developed APIs.\",\"extra\":true"), }; foreach (var assessment in invalid) Assert.False(CvAiGrounding.IsSupported("Built APIs.", "Developed APIs.", assessment)); } [Theory] [InlineData("", "supported", false)] [InlineData("Job requires API leadership.", "supported", false)] [InlineData("Built APIs.", "unknown", false)] [InlineData("Built APIs.", "unsupported", false)] [InlineData("Built APIs.", "supported", true)] public void Only_exact_candidate_quotes_and_supported_claims_are_accepted(string quote, string verdict, bool expected) => Assert.Equal(expected, CvAiGrounding.IsSupported("Built APIs.", "Developed APIs.", Assessment("Developed APIs.", verdict, quote))); [Fact] public void Missing_duplicate_or_wrong_proposal_coverage_fails_closed() { const string source = "Built APIs.\nUsed SQL."; const string proposal = "Developed APIs.\nWorked with SQL."; Assert.False(CvAiGrounding.IsSupported(source, proposal, Assessment("Developed APIs.", "supported", "Built APIs."))); var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = new[] { new { text = "Developed APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } }, new { text = "Developed APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } } } }); Assert.False(CvAiGrounding.IsSupported(source, proposal, assessment)); } [Fact] public void Document_json_covers_each_value_and_ignores_routing_ids_for_numeric_evidence() { const string source = "[{\"id\":\"job-50\",\"title\":\"Developer\",\"items\":[\"Built APIs.\"]}]"; const string proposal = "[{\"id\":\"job-50\",\"title\":\"Utvikler\",\"items\":[\"Utviklet API-er.\"]}]"; var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = new[] { new { text = "Utvikler", verdict = "supported", sourceQuotes = new[] { "Developer" } }, new { text = "Utviklet API-er.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } } } }); Assert.True(CvAiGrounding.IsSupported(source, proposal, assessment)); Assert.False(CvAiGrounding.IsSupported(source, "Managed 50 engineers.", Assessment("Managed 50 engineers.", "supported", "Built APIs."))); Assert.False(CvAiGrounding.IsSupported(source, "[{\"items\":[\"Developed APIs.\"]}]", Assessment("Developed APIs.", "supported", "Built APIs."))); } [Fact] public void Numeric_unit_changes_fail_even_when_the_reviewer_approves() => Assert.False(CvAiGrounding.IsSupported("Worked with 30 clients.", "Improved revenue 30%.", Assessment("Improved revenue 30%.", "supported", "Worked with 30 clients."))); [Theory] [InlineData("Built APIs.", "Maintained SQL.", "Developed APIs.")] [InlineData("Managed 30 clients.", "Managed 10 clients.", "Managed 30 clients.")] public void A_document_claim_cannot_borrow_facts_or_metrics_from_another_entry(string first, string second, string misplaced) { string Blocks(string a, string b) => JsonSerializer.Serialize(new[] { new { id = "entry:experience:a", items = new[] { a } }, new { id = "entry:experience:b", items = new[] { b } } }); var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = new[] { new { text = first, verdict = "supported", sourceQuotes = new[] { first } }, new { text = misplaced, verdict = "supported", sourceQuotes = new[] { first } } } }); Assert.False(CvAiGrounding.IsSupported(Blocks(first, second), Blocks(first, misplaced), assessment)); } [Fact] public void Repeated_grounded_wording_in_separate_entries_remains_valid() { const string source = "[{\"id\":\"entry:experience:a\",\"items\":[\"Built APIs.\"]},{\"id\":\"entry:experience:b\",\"items\":[\"Built APIs.\"]}]"; const string proposal = "[{\"id\":\"entry:experience:a\",\"items\":[\"Developed APIs.\"]},{\"id\":\"entry:experience:b\",\"items\":[\"Developed APIs.\"]}]"; var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = Enumerable.Range(0, 2).Select(_ => new { text = "Developed APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } }) }); Assert.True(CvAiGrounding.IsSupported(source, proposal, assessment)); } [Fact] public void Document_summary_can_consolidate_known_entry_facts() { const string source = "[{\"id\":\"entry:experience:a\",\"items\":[\"Built APIs.\"]},{\"id\":\"section:summary\",\"items\":[\"Software developer.\"]}]"; const string proposal = "[{\"id\":\"entry:experience:a\",\"items\":[\"Built APIs.\"]},{\"id\":\"section:summary\",\"items\":[\"Developed APIs.\"]}]"; var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = new[] { new { text = "Built APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } }, new { text = "Developed APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } } } }); Assert.True(CvAiGrounding.IsSupported(source, proposal, assessment)); } private static string Assessment(string proposal, string verdict, string quote) => JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = new[] { new { text = proposal, verdict, sourceQuotes = new[] { quote } } } }); }