jobtrackingapp/JobTrackerApi.Tests/CvAiGroundingTests.cs

using System.Text.Json;
using JobTrackerApi.Services;
using Xunit;

namespace JobTrackerApi.Tests;

public sealed class CvAiGroundingTests
{
    public static IEnumerable<object[]> Corpus()
    {
        using var document = JsonDocument.Parse(File.ReadAllText(Path.Combine(
            AppContext.BaseDirectory, "..", "..", "..", "Fixtures", "AiEvaluation", "cv-rewrite-grounding.json")));
        foreach (var value in document.RootElement.GetProperty("cases").EnumerateArray()) yield return new object[] { value.Clone() };
    }

    [Theory]
    [MemberData(nameof(Corpus))]
    public void Synthetic_semantic_assessments_enforce_the_grounding_contract(JsonElement fixture)
    {
        var source = fixture.GetProperty("source").GetString()!;
        var proposal = fixture.GetProperty("proposal").GetString()!;
        var verdict = fixture.GetProperty("verdict").GetString()!;
        // These are controlled evaluator judgments, not live-model factuality measurements.
        var assessment = JsonSerializer.Serialize(new { verdict, coverageComplete = true,
            claims = new[] { new { text = proposal, verdict, sourceQuotes = new[] { source } } } });
        Assert.Equal(fixture.GetProperty("expectedSupported").GetBoolean(), CvAiGrounding.IsSupported(source, proposal, assessment));
    }

    [Theory]
    [InlineData("not JSON")]
    [InlineData("{}")]
    [InlineData("{\"verdict\":\"supported\",\"coverageComplete\":false,\"claims\":[]}")]
    [InlineData("{\"verdict\":\"unknown\",\"coverageComplete\":true,\"claims\":[]}")]
    [InlineData("{\"verdict\":\"SUPPORTED\",\"coverageComplete\":true,\"claims\":[]}")]
    [InlineData("{\"verdict\":\"supported\",\"coverageComplete\":true,\"claims\":[],\"extra\":true}")]
    public void Missing_malformed_unknown_or_extra_assessments_fail_closed(string assessment)
        => Assert.False(CvAiGrounding.IsSupported("Built APIs.", "Developed APIs.", assessment));

    [Fact]
    public void Schema_corruption_fails_even_with_valid_coverage_and_quotes()
    {
        var valid = Assessment("Developed APIs.", "supported", "Built APIs.");
        Assert.True(CvAiGrounding.IsSupported("Built APIs.", "Developed APIs.", valid));
        var invalid = new[] {
            valid.Replace("\"coverageComplete\":true", "\"coverageComplete\":false"),
            valid.Replace("\"coverageComplete\":true,", ""),
            valid.Replace("\"coverageComplete\":true", "\"coverageComplete\":\"true\""),
            valid.Replace("\"supported\"", "\"unknown\""),
            valid.Insert(valid.Length - 1, ",\"extra\":true"),
            valid.Insert(valid.Length - 1, ",\"verdict\":\"supported\""),
            valid.Replace("\"sourceQuotes\":[\"Built APIs.\"]", "\"sourceQuotes\":[]"),
            valid.Replace("\"sourceQuotes\":[\"Built APIs.\"]", "\"sourceQuotes\":[\"Built APIs.\",\"Job requires SQL.\"]"),
            valid.Replace("\"text\":\"Developed APIs.\"", "\"text\":\"Developed APIs.\",\"extra\":true"),
        };
        foreach (var assessment in invalid) Assert.False(CvAiGrounding.IsSupported("Built APIs.", "Developed APIs.", assessment));
    }

    [Theory]
    [InlineData("", "supported", false)]
    [InlineData("Job requires API leadership.", "supported", false)]
    [InlineData("Built APIs.", "unknown", false)]
    [InlineData("Built APIs.", "unsupported", false)]
    [InlineData("Built APIs.", "supported", true)]
    public void Only_exact_candidate_quotes_and_supported_claims_are_accepted(string quote, string verdict, bool expected)
        => Assert.Equal(expected, CvAiGrounding.IsSupported("Built APIs.", "Developed APIs.", Assessment("Developed APIs.", verdict, quote)));

    [Fact]
    public void Missing_duplicate_or_wrong_proposal_coverage_fails_closed()
    {
        const string source = "Built APIs.\nUsed SQL.";
        const string proposal = "Developed APIs.\nWorked with SQL.";
        Assert.False(CvAiGrounding.IsSupported(source, proposal, Assessment("Developed APIs.", "supported", "Built APIs.")));
        var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true,
            claims = new[] { new { text = "Developed APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } },
                new { text = "Developed APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } } } });
        Assert.False(CvAiGrounding.IsSupported(source, proposal, assessment));
    }

    [Fact]
    public void Document_json_covers_each_value_and_ignores_routing_ids_for_numeric_evidence()
    {
        const string source = "[{\"id\":\"job-50\",\"title\":\"Developer\",\"items\":[\"Built APIs.\"]}]";
        const string proposal = "[{\"id\":\"job-50\",\"title\":\"Utvikler\",\"items\":[\"Utviklet API-er.\"]}]";
        var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true,
            claims = new[] { new { text = "Utvikler", verdict = "supported", sourceQuotes = new[] { "Developer" } },
                new { text = "Utviklet API-er.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } } } });
        Assert.True(CvAiGrounding.IsSupported(source, proposal, assessment));
        Assert.False(CvAiGrounding.IsSupported(source, "Managed 50 engineers.", Assessment("Managed 50 engineers.", "supported", "Built APIs.")));
        Assert.False(CvAiGrounding.IsSupported(source, "[{\"items\":[\"Developed APIs.\"]}]", Assessment("Developed APIs.", "supported", "Built APIs.")));
    }

    [Fact]
    public void Numeric_unit_changes_fail_even_when_the_reviewer_approves()
        => Assert.False(CvAiGrounding.IsSupported("Worked with 30 clients.", "Improved revenue 30%.", Assessment("Improved revenue 30%.", "supported", "Worked with 30 clients.")));

    [Theory]
    [InlineData("Built APIs.", "Maintained SQL.", "Developed APIs.")]
    [InlineData("Managed 30 clients.", "Managed 10 clients.", "Managed 30 clients.")]
    public void A_document_claim_cannot_borrow_facts_or_metrics_from_another_entry(string first, string second, string misplaced)
    {
        string Blocks(string a, string b) => JsonSerializer.Serialize(new[] {
            new { id = "entry:experience:a", items = new[] { a } },
            new { id = "entry:experience:b", items = new[] { b } } });
        var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = new[] {
            new { text = first, verdict = "supported", sourceQuotes = new[] { first } },
            new { text = misplaced, verdict = "supported", sourceQuotes = new[] { first } } } });
        Assert.False(CvAiGrounding.IsSupported(Blocks(first, second), Blocks(first, misplaced), assessment));
    }

    [Fact]
    public void Repeated_grounded_wording_in_separate_entries_remains_valid()
    {
        const string source = "[{\"id\":\"entry:experience:a\",\"items\":[\"Built APIs.\"]},{\"id\":\"entry:experience:b\",\"items\":[\"Built APIs.\"]}]";
        const string proposal = "[{\"id\":\"entry:experience:a\",\"items\":[\"Developed APIs.\"]},{\"id\":\"entry:experience:b\",\"items\":[\"Developed APIs.\"]}]";
        var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true,
            claims = Enumerable.Range(0, 2).Select(_ => new { text = "Developed APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } }) });
        Assert.True(CvAiGrounding.IsSupported(source, proposal, assessment));
    }

    [Fact]
    public void Document_summary_can_consolidate_known_entry_facts()
    {
        const string source = "[{\"id\":\"entry:experience:a\",\"items\":[\"Built APIs.\"]},{\"id\":\"section:summary\",\"items\":[\"Software developer.\"]}]";
        const string proposal = "[{\"id\":\"entry:experience:a\",\"items\":[\"Built APIs.\"]},{\"id\":\"section:summary\",\"items\":[\"Developed APIs.\"]}]";
        var assessment = JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = new[] {
            new { text = "Built APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } },
            new { text = "Developed APIs.", verdict = "supported", sourceQuotes = new[] { "Built APIs." } } } });
        Assert.True(CvAiGrounding.IsSupported(source, proposal, assessment));
    }

    private static string Assessment(string proposal, string verdict, string quote) => JsonSerializer.Serialize(new {
        verdict = "supported", coverageComplete = true, claims = new[] { new { text = proposal, verdict, sourceQuotes = new[] { quote } } } });
}