using System.Text.Json; using System.Security.Claims; using JobTrackerApi.Controllers; using JobTrackerApi.Tests.TestSupport; using JobTrackerApi.Data; using JobTrackerApi.Models; using JobTrackerApi.Services; using Microsoft.AspNetCore.DataProtection; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Moq; using Xunit; namespace JobTrackerApi.Tests; public sealed class CvAiReadinessTests { private const string Source = "Built and maintained internal web applications."; private const string Paraphrase = "Developed and maintained internal web applications."; private sealed class Fixture : IAsyncDisposable { public readonly SqliteConnection Connection = new("Data Source=:memory:"); public readonly Mock Ai = new(); public readonly IDataProtectionProvider Protection = new EphemeralDataProtectionProvider(); public JobTrackerContext Db = null!; public CareerProfileService Career = null!; public CvVariantService Variants = null!; public CvAiSuggestionService Suggestions = null!; public CvVariant Cv = null!; public StructuredCvProfile Profile = new() { Contact = new() { FullName = "Åse Synthetic", Headline = Source }, Summary = { Source } }; public async Task Open() { await Connection.OpenAsync(); var current = new Mock(); current.SetupGet(value => value.UserId).Returns("owner"); Db = new(new DbContextOptionsBuilder().UseSqlite(Connection).Options, current.Object); await Db.Database.EnsureCreatedAsync(); Career = new(Db); await Career.SaveVersionAsync("owner", Profile, "manual", default); Variants = new(Db, Career, new ThemedCvRenderer()); Cv = await Variants.CreateAsync("owner", "Synthetic CV", null, new CvVariantSettings(), default); Suggestions = new(Db, Career, Variants, Ai.Object, Protection); Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(Paraphrase)); Ai.Setup(value => value.EvaluateCvRewriteAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((string candidate, string proposal, string? context, CancellationToken ct) => new AiGenerationResult(Assessment(proposal, Source))); } public Task Generate(string? context = null, string target = "section:summary") => Suggestions.GenerateAsync("owner", Cv.Id, 1, target, "improve", Source, context, "Rewrite supported facts only.", default); public CvVariantController Controller(bool reviewAvailable = true) { var user = new ApplicationUser { Id = "owner" }; var users = TestHostFactory.CreateUserManager(user); users.Setup(value => value.GetUserAsync(It.IsAny())).ReturnsAsync(user); return new(users.Object, Variants, Mock.Of(), Ai.Object, suggestions: reviewAvailable ? Suggestions : null) { ControllerContext = new() { HttpContext = new DefaultHttpContext { User = new ClaimsPrincipal(new ClaimsIdentity(new[] { new Claim(ClaimTypes.NameIdentifier, "owner") }, "local")) } } }; } public async Task AssertUnchanged() { Db.ChangeTracker.Clear(); var saved = await Db.CvVariants.SingleAsync(); Assert.Equal(1, saved.Version); Assert.Null(CvVariantSettingsJson.Deserialize(saved.SettingsJson).Headline); Assert.Empty(CvVariantSettingsJson.Deserialize(saved.SettingsJson).Sections); Assert.Single(await Db.CvVariantVersions.ToListAsync()); Assert.Equal(Source, (await Career.LoadStructuredForOwnerAsync("owner", default)).Contact.Headline); Assert.Single(await Db.CareerProfileVersions.ToListAsync()); } public async ValueTask DisposeAsync() { await Db.DisposeAsync(); await Connection.DisposeAsync(); } } private static string Assessment(string proposal, string source, string verdict = "supported") => JsonSerializer.Serialize(new { verdict, coverageComplete = true, claims = new[] { new { text = proposal, verdict, sourceQuotes = new[] { source } } } }); [Theory] [InlineData(Paraphrase)] [InlineData("Maintained internal web applications.")] public async Task Grounded_rewriting_is_applied_only_on_accept_and_uses_normal_history(string rewrite) { await using var fixture = new Fixture(); await fixture.Open(); fixture.Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(rewrite)); var proposal = await fixture.Generate(); await fixture.AssertUnchanged(); // generation has no content/history mutation var accepted = await fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id, proposal.ProposalToken, null, default); Assert.Equal(2, accepted.Version); Assert.Equal(rewrite, CvVariantSettingsJson.Deserialize(accepted.SettingsJson).Sections.Single().Items!.Single()); Assert.Equal("ai:improve", (await fixture.Db.CvVariantVersions.OrderBy(value => value.Version).LastAsync()).Source); Assert.Equal(Source, (await fixture.Career.LoadStructuredForOwnerAsync("owner", default)).Summary.Single()); Assert.Equal(Source, (await fixture.Career.LoadStructuredForOwnerAsync("owner", default)).Contact.Headline); var historical = await fixture.Db.CvVariantVersions.SingleAsync(value => value.Version == 1); Assert.Empty(CvVariantSettingsJson.Deserialize(historical.SettingsJson).Sections); var replay = await Assert.ThrowsAsync(() => fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id, proposal.ProposalToken, null, default)); Assert.Equal(409, replay.Status); } [Theory] [InlineData("Experienced with Kubernetes.")] [InlineData("Senior engineer at Microsoft.")] [InlineData("Completed a master's degree and Azure certification.")] [InlineData("Managed 50 engineers at Microsoft and increased revenue 300%.")] [InlineData("Built internal web applications and led international security audits.")] [InlineData("Flytende fransk og sertifisert prosjektleder.")] [InlineData("Worked at Microsoft from 2018 to 2025.")] public async Task Fabricated_or_mixed_provider_output_cannot_obtain_an_applicable_proposal(string invented) { await using var fixture = new Fixture(); await fixture.Open(); fixture.Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(invented)); fixture.Ai.Setup(value => value.EvaluateCvRewriteAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(Assessment(invented, Source, "unsupported"))); var error = await Assert.ThrowsAsync(() => fixture.Generate()); Assert.Equal(422, error.Status); Assert.Equal("cv_ai_grounding_unproven", error.Code); await fixture.AssertUnchanged(); } [Fact] public async Task Job_requirements_are_context_not_candidate_evidence() { await using var fixture = new Fixture(); await fixture.Open(); const string requirement = "Kubernetes required"; const string invented = "Experienced with Kubernetes"; fixture.Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(invented)); // Even a 'supported' verdict cannot cite the advert as candidate evidence. fixture.Ai.Setup(value => value.EvaluateCvRewriteAsync(It.IsAny(), invented, requirement, It.IsAny())) .ReturnsAsync(new AiGenerationResult(Assessment(invented, requirement))); var error = await Assert.ThrowsAsync(() => fixture.Generate(requirement)); Assert.Equal(422, error.Status); fixture.Ai.Verify(value => value.EvaluateCvRewriteAsync(It.Is(source => !source.Contains("Kubernetes")), invented, requirement, It.IsAny()), Times.Once); await fixture.AssertUnchanged(); } [Theory] [InlineData(false)] [InlineData(true)] public async Task Pending_proposal_cannot_overwrite_newer_CV_or_canonical_source(bool canonical) { await using var fixture = new Fixture(); await fixture.Open(); var proposal = await fixture.Generate(); if (canonical) { fixture.Profile.Summary = ["Newer canonical user content"]; await fixture.Career.SaveVersionAsync("owner", fixture.Profile, "manual", default); } else await fixture.Variants.SaveAsync("owner", fixture.Cv.Id, null, new CvVariantSettings { Headline = "Newer CV user content" }, "manual", 1, default); var historyCount = await fixture.Db.CvVariantVersions.CountAsync(); var error = await Assert.ThrowsAsync(() => fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id, proposal.ProposalToken, null, default)); Assert.Equal(409, error.Status); Assert.Equal("cv_ai_source_changed", error.Code); fixture.Db.ChangeTracker.Clear(); Assert.Equal(historyCount, await fixture.Db.CvVariantVersions.CountAsync()); var saved = CvVariantSettingsJson.Deserialize((await fixture.Db.CvVariants.SingleAsync()).SettingsJson); Assert.Empty(saved.Sections); if (canonical) Assert.Equal("Newer canonical user content", (await fixture.Career.LoadStructuredForOwnerAsync("owner", default)).Summary.Single()); else Assert.Equal("Newer CV user content", saved.Headline); } [Fact] public async Task Source_change_during_generation_does_not_issue_a_stale_proposal() { await using var fixture = new Fixture(); await fixture.Open(); fixture.Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(async () => { await fixture.Variants.SaveAsync("owner", fixture.Cv.Id, null, new CvVariantSettings { Headline = "Newer" }, "manual", 1, default); return new AiGenerationResult(Paraphrase); }); var error = await Assert.ThrowsAsync(() => fixture.Generate()); Assert.Equal(409, error.Status); Assert.Equal(2, await fixture.Db.CvVariantVersions.CountAsync()); } [Fact] public async Task Canonical_change_during_edited_proposal_review_is_checked_again_before_save() { await using var fixture = new Fixture(); await fixture.Open(); var proposal = await fixture.Generate(); const string shortened = "Maintained internal web applications."; fixture.Ai.Setup(value => value.EvaluateCvRewriteAsync(It.IsAny(), shortened, It.IsAny(), It.IsAny())) .Returns(async () => { fixture.Profile.Summary = ["Newer canonical content"]; await fixture.Career.SaveVersionAsync("owner", fixture.Profile, "manual", default); return new AiGenerationResult(Assessment(shortened, Source)); }); var error = await Assert.ThrowsAsync(() => fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id, proposal.ProposalToken, shortened, default)); Assert.Equal(409, error.Status); fixture.Db.ChangeTracker.Clear(); Assert.Single(await fixture.Db.CvVariantVersions.ToListAsync()); Assert.Equal(1, (await fixture.Db.CvVariants.SingleAsync()).Version); Assert.Equal("Newer canonical content", (await fixture.Career.LoadStructuredForOwnerAsync("owner", default)).Summary.Single()); } [Fact] public async Task Whitespace_consistency_check_does_not_block_supported_rewriting() { await using var fixture = new Fixture(); await fixture.Open(); var proposal = await fixture.Suggestions.GenerateAsync("owner", fixture.Cv.Id, 1, "section:summary", "improve", " " + Source + "\r\n\n", null, "Rewrite", default); Assert.Equal(Source, proposal.Original); await fixture.AssertUnchanged(); } [Fact] public async Task Whole_document_proposal_cannot_review_multiple_headline_items_then_drop_one_on_apply() { await using var fixture = new Fixture(); await fixture.Open(); const string invalid = "[{\"id\":\"headline\",\"items\":[\"First\",\"Second\"]},{\"id\":\"section:summary\",\"title\":\"Summary\",\"items\":[]}]"; fixture.Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), AiGenerationOutputMode.CvBlocksJson, It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(invalid)); var error = await Assert.ThrowsAsync(() => fixture.Suggestions.GenerateAsync("owner", fixture.Cv.Id, 1, null, "document-improve", null, null, "Rewrite", default)); Assert.Equal("cv_ai_proposal_invalid", error.Code); fixture.Ai.Verify(value => value.EvaluateCvRewriteAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); await fixture.AssertUnchanged(); } [Fact] public async Task Edited_proposal_is_re_evaluated_and_unsupported_edit_has_no_mutation() { await using var fixture = new Fixture(); await fixture.Open(); var proposal = await fixture.Generate(); const string invented = "Invented job at Microsoft"; fixture.Ai.Setup(value => value.EvaluateCvRewriteAsync(It.IsAny(), invented, It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(Assessment(invented, Source, "unsupported"))); var error = await Assert.ThrowsAsync(() => fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id, proposal.ProposalToken, invented, default)); Assert.Equal(422, error.Status); await fixture.AssertUnchanged(); } [Fact] public async Task Tampered_token_cannot_bypass_grounding_or_select_another_target() { await using var fixture = new Fixture(); await fixture.Open(); var proposal = await fixture.Generate(); var error = await Assert.ThrowsAsync(() => fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id, "forged-grounded-true-" + proposal.ProposalToken, null, default)); Assert.Equal(400, error.Status); await fixture.AssertUnchanged(); } [Fact] public async Task Protected_proposal_is_bound_to_owner_and_intended_CV() { await using var fixture = new Fixture(); await fixture.Open(); var proposal = await fixture.Generate(); var ownerError = await Assert.ThrowsAsync(() => fixture.Suggestions.ApplyAsync("other-account", fixture.Cv.Id, proposal.ProposalToken, null, default)); var cvError = await Assert.ThrowsAsync(() => fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id + 1, proposal.ProposalToken, null, default)); Assert.Equal(404, ownerError.Status); Assert.Equal(404, cvError.Status); var generation = await Assert.ThrowsAsync(() => fixture.Suggestions.GenerateAsync("other-account", fixture.Cv.Id, 1, "section:summary", "improve", null, null, "Rewrite", default)); Assert.Equal(404, generation.Status); await fixture.AssertUnchanged(); } [Fact] public async Task Evaluator_unavailable_fails_closed_and_provider_error_never_changes_content() { await using var fixture = new Fixture(); await fixture.Open(); fixture.Ai.Setup(value => value.EvaluateCvRewriteAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((AiGenerationResult?)null); Assert.Equal(422, (await Assert.ThrowsAsync(() => fixture.Generate())).Status); fixture.Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ThrowsAsync(new AiGenerationException("provider_unavailable", "Synthetic outage", true)); await Assert.ThrowsAsync(() => fixture.Generate()); await fixture.AssertUnchanged(); } [Fact] public async Task Outdated_unbound_API_request_fails_closed_before_provider_generation() { await using var fixture = new Fixture(); await fixture.Open(); var response = await fixture.Controller().AiAssist(new("improve", Source, null, null, "en"), default); Assert.Equal(428, Assert.IsType(response.Result).StatusCode); fixture.Ai.Verify(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); await fixture.AssertUnchanged(); } [Theory] [InlineData(false)] [InlineData(true)] public async Task Unavailable_review_service_blocks_generation_and_apply_without_side_effects(bool apply) { await using var fixture = new Fixture(); await fixture.Open(); var controller = fixture.Controller(reviewAvailable: false); var response = apply ? Assert.IsType((await controller.ApplyAi(fixture.Cv.Id, new("unused-proposal", Paraphrase), default)).Result) : Assert.IsType((await controller.AiAssist(new("improve", Source, null, null, "en", fixture.Cv.Id, 1, "section:summary"), default)).Result); Assert.Equal(503, response.StatusCode); Assert.Equal("cv_ai_unavailable", JsonSerializer.SerializeToElement(response.Value).GetProperty("code").GetString()); fixture.Ai.VerifyNoOtherCalls(); await fixture.AssertUnchanged(); } [Fact] public async Task Confirmed_provider_fabrication_reproduction_is_now_an_API_rejection() { await using var fixture = new Fixture(); await fixture.Open(); const string invented = "Managed 50 engineers at Microsoft and increased revenue 300%."; fixture.Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(invented)); // Numerics are rejected independently, even if the evaluator asserts support. fixture.Ai.Setup(value => value.EvaluateCvRewriteAsync(It.IsAny(), invented, It.IsAny(), It.IsAny())) .ReturnsAsync(new AiGenerationResult(Assessment(invented, Source))); var response = await fixture.Controller().AiAssist(new("improve", Source, null, null, "en", fixture.Cv.Id, 1, "section:summary"), default); Assert.Equal(422, Assert.IsType(response.Result).StatusCode); await fixture.AssertUnchanged(); } [Theory] [InlineData("ai:improve")] [InlineData(" AI:grammar ")] public async Task Authored_save_cannot_forge_server_AI_provenance(string source) { await using var fixture = new Fixture(); await fixture.Open(); var response = await fixture.Controller().Save(fixture.Cv.Id, new(null, new CvVariantSettings { Headline = "Forged approval" }, source, 1), default); Assert.IsType(response.Result); await fixture.AssertUnchanged(); } [Fact] public async Task Confirmed_stale_apply_reproduction_is_now_an_API_conflict() { await using var fixture = new Fixture(); await fixture.Open(); var proposal = await fixture.Generate(); await fixture.Variants.SaveAsync("owner", fixture.Cv.Id, null, new CvVariantSettings { Headline = "Newer user content" }, "manual", 1, default); var response = await fixture.Controller().ApplyAi(fixture.Cv.Id, new(proposal.ProposalToken), default); Assert.Equal(409, Assert.IsType(response.Result).StatusCode); fixture.Db.ChangeTracker.Clear(); Assert.Equal("Newer user content", CvVariantSettingsJson.Deserialize((await fixture.Db.CvVariants.SingleAsync()).SettingsJson).Headline); Assert.Equal(2, await fixture.Db.CvVariantVersions.CountAsync()); } [Fact] public async Task Omitted_content_remains_editable_without_becoming_document_evidence() { await using var fixture = new Fixture(); await fixture.Open(); await fixture.Variants.SaveAsync("owner", fixture.Cv.Id, null, new CvVariantSettings { Sections = { new() { Key = "summary", Hidden = true } } }, "manual", 1, default); var proposal = await fixture.Suggestions.GenerateAsync("owner", fixture.Cv.Id, 2, "section:summary", "improve", Source, null, "Rewrite", default); var saved = await fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id, proposal.ProposalToken, null, default); var section = CvVariantSettingsJson.Deserialize(saved.SettingsJson).Sections.Single(); Assert.True(section.Hidden); Assert.Equal(Paraphrase, section.Items!.Single()); Assert.Equal(Source, (await fixture.Career.LoadStructuredForOwnerAsync("owner", default)).Summary.Single()); } [Fact] public async Task Whole_document_accept_maps_known_blocks_only_and_preserves_the_rest_of_the_lens() { await using var fixture = new Fixture(); await fixture.Open(); fixture.Profile.Jobs = [new() { Id = "experience-key", Title = "Engineer", Company = "Nordlys", Bullets = { Source } }]; await fixture.Career.SaveVersionAsync("owner", fixture.Profile, "manual", default); await fixture.Variants.SaveAsync("owner", fixture.Cv.Id, null, new CvVariantSettings { AccentColor = "#123456", PageMarginMm = 22, Sections = { new() { Key = "experience" }, new() { Key = "summary" } } }, "manual", 1, default); fixture.Ai.Setup(value => value.GenerateSectionWithMetadataAsync(It.IsAny(), It.IsAny(), AiGenerationOutputMode.CvBlocksJson, It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((string instruction, string text, AiGenerationOutputMode mode, int max, int min, CancellationToken ct) => new AiGenerationResult(text.Replace(Source, Paraphrase, StringComparison.Ordinal))); fixture.Ai.Setup(value => value.EvaluateCvRewriteAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .ReturnsAsync((string candidate, string proposal, string? context, CancellationToken ct) => { using var parsed = JsonDocument.Parse(proposal); var values = parsed.RootElement.EnumerateArray().SelectMany(block => block.EnumerateObject() .Where(property => property.Name != "id").SelectMany(property => property.Value.ValueKind == JsonValueKind.Array ? property.Value.EnumerateArray().Select(item => item.GetString()!) : new[] { property.Value.GetString()! })); return new AiGenerationResult(JsonSerializer.Serialize(new { verdict = "supported", coverageComplete = true, claims = values.Select(value => new { text = value, verdict = "supported", sourceQuotes = new[] { value == Paraphrase ? Source : value } }) })); }); var proposal = await fixture.Suggestions.GenerateAsync("owner", fixture.Cv.Id, 2, null, "document-improve", null, null, "Rewrite", default); var saved = await fixture.Suggestions.ApplyAsync("owner", fixture.Cv.Id, proposal.ProposalToken, null, default); var settings = CvVariantSettingsJson.Deserialize(saved.SettingsJson); Assert.Equal(3, saved.Version); Assert.Equal(Paraphrase, settings.Headline); Assert.Equal(Paraphrase, settings.Overrides["experience-key"].Bullets!.Single()); Assert.Equal("Engineer", settings.Overrides["experience-key"].Title); Assert.Equal("Nordlys", settings.Overrides["experience-key"].Subtitle); Assert.Equal(new[] { "experience", "summary" }, settings.Sections.Select(value => value.Key)); Assert.Equal("#123456", settings.AccentColor); Assert.Equal(22, settings.PageMarginMm); Assert.Equal(Source, (await fixture.Career.LoadStructuredForOwnerAsync("owner", default)).Jobs.Single().Bullets.Single()); } }