jobtrackingapp/JobTrackerApi.Tests/CvProcessingOperationTests.cs

using System.Security.Claims;
using System.Text;
using JobTrackerApi.Controllers;
using JobTrackerApi.Data;
using JobTrackerApi.Models;
using JobTrackerApi.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Moq;
using Xunit;

namespace JobTrackerApi.Tests;

public sealed class CvProcessingOperationTests
{
    [Fact]
    public async Task Worker_that_lost_its_lease_cannot_overwrite_a_recovered_successful_review()
    {
        await using var fixture = await Fixture.CreateAsync();
        await fixture.SeedUserAsync();
        var entered = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
        fixture.DocumentExtraction = async () => { entered.TrySetResult(); await release.Task; return new("Skills\nC#", false, "application/pdf", 1, 9, "synthetic.pdf", new[] { new CvImportBlock("b1", "Skills\nC#", "paragraph", 1) }, Array.Empty<string>(), "docling-native-pdf-v1"); };
        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            var bytes = Encoding.UTF8.GetBytes("%PDF-1.4\nSynthetic fixture");
            Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Import(new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "synthetic.pdf") { Headers = new HeaderDictionary(), ContentType = "application/pdf" }));
        }
        var worker = fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default);
        try
        {
            await entered.Task.WaitAsync(TimeSpan.FromSeconds(10));
            await using var recovered = fixture.Provider.CreateAsyncScope();
            var db = recovered.ServiceProvider.GetRequiredService<JobTrackerContext>();
            var now = DateTime.UtcNow;
            await db.UserOperations.IgnoreQueryFilters().ExecuteUpdateAsync(s => s.SetProperty(o => o.Status, OperationStatuses.Succeeded).SetProperty(o => o.LeaseToken, (string?)null).SetProperty(o => o.CompletedAtUtc, now));
            await db.CvExtractionRuns.IgnoreQueryFilters().ExecuteUpdateAsync(s => s.SetProperty(r => r.Status, "pending_review").SetProperty(r => r.StructuredProfileJson, "{\"skills\":[\"Recovered winner\"]}"));
        }
        finally { release.TrySetResult(); }
        Assert.True(await worker.WaitAsync(TimeSpan.FromSeconds(10)));
        await using var verify = fixture.Provider.CreateAsyncScope();
        var winner = await verify.ServiceProvider.GetRequiredService<JobTrackerContext>().CvExtractionRuns.IgnoreQueryFilters().SingleAsync();
        Assert.Equal("pending_review", winner.Status);
        Assert.Contains("Recovered winner", winner.StructuredProfileJson);
    }
    [Fact]
    public async Task Document_failure_message_is_bounded_and_does_not_crash_the_worker_or_disclose_source()
    {
        await using var fixture = await Fixture.CreateAsync();
        await fixture.SeedUserAsync();
        fixture.ExtractionFailure = new InvalidOperationException("PRIVATE_SOURCE " + new string('x', 2000));
        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            var bytes = Encoding.UTF8.GetBytes("%PDF-1.4\nSynthetic fixture");
            Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Import(new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "synthetic.pdf") { Headers = new HeaderDictionary(), ContentType = "application/pdf" }));
        }
        Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
        await using var verify = fixture.Provider.CreateAsyncScope();
        var operation = await verify.ServiceProvider.GetRequiredService<JobTrackerContext>().UserOperations.IgnoreQueryFilters().SingleAsync();
        Assert.Equal(OperationStatuses.Failed, operation.Status);
        Assert.InRange(operation.FailureMessage!.Length, 1, 512);
        Assert.DoesNotContain("PRIVATE_SOURCE", operation.FailureMessage);
    }
    [Fact]
    public async Task Structured_document_worker_publishes_review_under_its_real_database_lease()
    {
        await using var fixture = await Fixture.CreateAsync();
        await fixture.SeedUserAsync();
        CvProcessingOperationResponse response;
        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            var bytes = Encoding.UTF8.GetBytes("%PDF-1.4\nSynthetic transport fixture");
            response = Response(await CreateController(scope.ServiceProvider).Import(new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "synthetic.pdf") { Headers = new HeaderDictionary(), ContentType = "application/pdf" }));
        }
        Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
        await using var verify = fixture.Provider.CreateAsyncScope();
        var db = verify.ServiceProvider.GetRequiredService<JobTrackerContext>();
        Assert.Equal("pending_review", (await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync()).Status);
        Assert.Equal(OperationStatuses.Succeeded, (await db.UserOperations.IgnoreQueryFilters().SingleAsync()).Status);
        Assert.Empty(await db.CareerProfiles.IgnoreQueryFilters().ToListAsync());
    }
    [Fact]
    public async Task Cancellation_of_an_expired_import_operation_cannot_resurrect_its_review()
    {
        await using var fixture = await Fixture.CreateAsync();
        await fixture.SeedUserAsync();
        await using var scope = fixture.Provider.CreateAsyncScope();
        using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
        var response = Response(await CreateController(scope.ServiceProvider).Upload(File()));
        var db = scope.ServiceProvider.GetRequiredService<JobTrackerContext>();
        var run = await db.CvExtractionRuns.SingleAsync();
        run.ParserVersion = ProfileCvController.DocumentParserVersion;
        run.Status = "expired";
        await db.SaveChangesAsync();
        Assert.True(await scope.ServiceProvider.GetRequiredService<UserOperationStore>().RequestCancellationAsync(response.Operation!.Id, default));
        await db.Entry(run).ReloadAsync();
        Assert.Equal("expired", run.Status);
    }
    [Fact]
    public async Task Upload_is_durable_idempotent_and_stops_at_review_gate()
    {
        await using var fixture = await Fixture.CreateAsync();
        await fixture.SeedUserAsync();

        CvProcessingOperationResponse first;
        CvProcessingOperationResponse duplicate;
        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            var controller = CreateController(scope.ServiceProvider);
            first = Response(await controller.Upload(File()));
            duplicate = Response(await controller.Upload(File()));
            var runs = Assert.IsType<OkObjectResult>((await controller.GetRuns()).Result);
            Assert.Equal(first.Operation?.Id, Assert.Single(Assert.IsAssignableFrom<IEnumerable<CvExtractionRunListItem>>(runs.Value)).Operation?.Id);
        }

        Assert.Equal(first.ExtractionRunId, duplicate.ExtractionRunId);
        Assert.Equal(first.Operation?.Id, duplicate.Operation?.Id);
        Assert.True(first.Created);
        Assert.False(duplicate.Created);

        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            var db = scope.ServiceProvider.GetRequiredService<JobTrackerContext>();
            Assert.Single(await db.CvExtractionRuns.IgnoreQueryFilters().ToListAsync());
            Assert.Single(await db.CvUploadArtifacts.IgnoreQueryFilters().ToListAsync());
            var operation = Assert.Single(await db.UserOperations.IgnoreQueryFilters().ToListAsync());
            Assert.Equal(CvProcessingQueue.TaskType, operation.TaskType);
            Assert.Equal(CvProcessingQueue.SubjectType, operation.SubjectType);
            Assert.DoesNotContain("Ada", operation.SubjectId ?? string.Empty, StringComparison.OrdinalIgnoreCase);
            var usage = Assert.Single(await db.AiUsageRecords.IgnoreQueryFilters().ToListAsync());
            Assert.Equal(operation.Id.ToString("D"), usage.SourceId);
            Assert.Equal(AiUsageMeter.ReservationFor(CvProcessingQueue.TaskType).EstimatedTokens, usage.EstimatedTokenCount);
        }

        Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
        Assert.False(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));

        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            var db = scope.ServiceProvider.GetRequiredService<JobTrackerContext>();
            var run = await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync();
            var operation = await db.UserOperations.IgnoreQueryFilters().SingleAsync();
            var user = await db.Users.SingleAsync();
            Assert.Equal("pending_review", run.Status);
            Assert.Equal(OperationStatuses.Succeeded, operation.Status);
            Assert.Equal($"/api/profile-cv/runs/{run.Id}/diff", operation.ResultReference);
            Assert.Null(user.ProfileCvStructureJson);
            Assert.Null(user.CurrentCvExtractionRunId);
            Assert.Equal("operation_succeeded", (await db.UserNotifications.IgnoreQueryFilters().SingleAsync()).Kind);
            Assert.Equal(AiUsageMeter.ReservationFor(CvProcessingQueue.TaskType).EstimatedTokens,
                (await db.AiUsageRecords.IgnoreQueryFilters().SingleAsync()).EstimatedTokenCount);
        }
    }

    [Fact]
    public async Task Retryable_provider_failure_keeps_run_queued_and_records_provenance()
    {
        await using var fixture = await Fixture.CreateAsync();
        await fixture.SeedUserAsync(profileCvText: "# Ada Lovelace\n\n## Skills\nC#");
        fixture.GenerationFailure = new AiGenerationException(
            "provider_unavailable",
            "The local provider is unavailable.",
            retryable: true,
            provider: "ollama",
            model: "qwen-test",
            routeReason: "local_primary");

        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Improve());
        }

        Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));

        await using var verification = fixture.Provider.CreateAsyncScope();
        var db = verification.ServiceProvider.GetRequiredService<JobTrackerContext>();
        var operation = await db.UserOperations.IgnoreQueryFilters().SingleAsync();
        var run = await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync();
        Assert.Equal(OperationStatuses.WaitingForRetry, operation.Status);
        Assert.Equal("provider_unavailable", operation.FailureCategory);
        Assert.Equal("ollama", operation.Provider);
        Assert.Equal("qwen-test", operation.Model);
        Assert.Equal("local_primary", operation.ProgressStage);
        Assert.Equal("queued", run.Status);
        Assert.Null(run.CompletedAtUtc);
    }

    [Fact]
    public async Task Cancellation_before_claim_and_retry_keep_extraction_history_in_sync()
    {
        await using var fixture = await Fixture.CreateAsync();
        await fixture.SeedUserAsync();

        Guid operationId;
        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Upload(File()));
            operationId = (await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().UserOperations.SingleAsync()).Id;
        }

        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            var store = scope.ServiceProvider.GetRequiredService<UserOperationStore>();
            Assert.True(await store.RequestCancellationAsync(operationId, default));

            var cancelledRun = await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().CvExtractionRuns.AsNoTracking().SingleAsync();
            Assert.Equal("cancelled", cancelledRun.Status);
            Assert.NotNull(cancelledRun.CompletedAtUtc);
        }

        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            Assert.True(await scope.ServiceProvider.GetRequiredService<UserOperationStore>().RetryAsync(operationId, default));
            var queuedRun = await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().CvExtractionRuns.AsNoTracking().SingleAsync();
            Assert.Equal("queued", queuedRun.Status);
            Assert.Null(queuedRun.ErrorMessage);
            Assert.Null(queuedRun.CompletedAtUtc);
        }

        Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
        await using var verification = fixture.Provider.CreateAsyncScope();
        Assert.Equal("pending_review", (await verification.ServiceProvider.GetRequiredService<JobTrackerContext>()
            .CvExtractionRuns.IgnoreQueryFilters().SingleAsync()).Status);
    }

    [Fact]
    public async Task Deadline_before_claim_fails_the_dormant_extraction_run()
    {
        await using var fixture = await Fixture.CreateAsync();
        await fixture.SeedUserAsync();

        await using (var scope = fixture.Provider.CreateAsyncScope())
        {
            using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
            Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Upload(File()));
            await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().UserOperations
                .ExecuteUpdateAsync(setters => setters.SetProperty(item => item.DeadlineAtUtc, DateTime.UtcNow.AddMinutes(-1)));
        }

        Assert.False(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
        await using var verification = fixture.Provider.CreateAsyncScope();
        var db = verification.ServiceProvider.GetRequiredService<JobTrackerContext>();
        Assert.Equal(OperationStatuses.Failed, (await db.UserOperations.IgnoreQueryFilters().SingleAsync()).Status);
        var run = await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync();
        Assert.Equal("failed", run.Status);
        Assert.Contains("deadline", run.ErrorMessage, StringComparison.OrdinalIgnoreCase);
        Assert.NotNull(run.CompletedAtUtc);
    }

    private static ProfileCvController CreateController(IServiceProvider services)
    {
        var controller = services.GetRequiredService<ProfileCvController>();
        controller.ControllerContext = new ControllerContext
        {
            HttpContext = new DefaultHttpContext
            {
                User = new ClaimsPrincipal(new ClaimsIdentity(
                    new[] { new Claim(ClaimTypes.NameIdentifier, "user-1") },
                    "test")),
            },
        };
        return controller;
    }

    private static CvProcessingOperationResponse Response(IActionResult result)
        => Assert.IsType<CvProcessingOperationResponse>(Assert.IsType<AcceptedResult>(result).Value);

    private static FormFile File()
    {
        const string text = "# Ada Lovelace\n\n## Professional Summary\nBuilt reliable analytical systems.\n\n## Skills\nC#\nSQL";
        var bytes = Encoding.UTF8.GetBytes(text);
        return new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "synthetic-cv.md")
        {
            Headers = new HeaderDictionary(),
            ContentType = "text/markdown",
        };
    }

    private sealed class Fixture : IAsyncDisposable
    {
        private readonly SqliteConnection _connection;
        private readonly string _tempRoot;
        public ServiceProvider Provider { get; }
        public AiGenerationException? GenerationFailure { get; set; }
        public Exception? ExtractionFailure { get; set; }
        public Func<Task<CvImportDocument>>? DocumentExtraction { get; set; }

        private Fixture(SqliteConnection connection, string tempRoot, ServiceProvider provider)
        {
            _connection = connection;
            _tempRoot = tempRoot;
            Provider = provider;
        }

        public static async Task<Fixture> CreateAsync()
        {
            var connection = new SqliteConnection("Data Source=:memory:");
            await connection.OpenAsync();
            var tempRoot = Path.Combine(Path.GetTempPath(), $"jobtracker-cv-operation-{Guid.NewGuid():N}");
            var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
            {
                ["Data:Root"] = tempRoot,
                ["Data:CvArtifactsRoot"] = Path.Combine(tempRoot, "CvArtifacts"),
                ["AiQueue:HeartbeatSeconds"] = "5",
                ["Ai:ExternalProcessingEnabled"] = "false",
            }).Build();
            var environment = new Mock<IHostEnvironment>();
            environment.SetupGet(item => item.ContentRootPath).Returns(tempRoot);
            Fixture? fixture = null;
            var summarizer = new Mock<ISummarizerService>();
            summarizer.Setup(item => item.ExtractCvDocumentAsync(It.IsAny<Stream>(), It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
                .Returns(async () => fixture?.ExtractionFailure is not null ? throw fixture.ExtractionFailure : fixture?.DocumentExtraction is not null ? await fixture.DocumentExtraction() : new CvImportDocument("Skills\nC#", false, "application/pdf", 1, 9, "synthetic.pdf", new[] { new CvImportBlock("b1", "Skills", "section_header", 1), new CvImportBlock("b2", "C#", "paragraph", 1) }, Array.Empty<string>(), "docling-native-pdf-v1"));
            summarizer.Setup(item => item.ExtractTextAsync(
                    It.IsAny<Stream>(), It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
                .ReturnsAsync(new AiTextExtractionResult(
                    "# Ada Lovelace\n\n## Professional Summary\nBuilt reliable analytical systems.\n\n## Skills\nC#\nSQL",
                    false,
                    "text/markdown",
                    null,
                    94,
                    "synthetic-cv.md"));
            summarizer.Setup(item => item.SummarizeSectionAsync(
                    It.Is<string>(instruction => instruction.Contains("structured JSON", StringComparison.Ordinal)),
                    It.IsAny<string>(), 3200, 900))
                .ReturnsAsync("""
                    {"version":"1","contact":{"fullName":"Ada Lovelace"},"summary":["Built reliable analytical systems."],"jobs":[],"education":[],"skills":["C#","SQL"],"languages":[],"interests":[],"otherSections":[]}
                    """);
            summarizer.Setup(item => item.GenerateSectionWithMetadataAsync(
                    It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
                .ReturnsAsync(() => fixture!.GenerationFailure is null
                    ? new AiGenerationResult("# Ada Lovelace\n\n## Skills\nC#", "ollama", "qwen-test", RouteReason: "local_primary")
                    : throw fixture.GenerationFailure);

            var services = new ServiceCollection();
            services.AddSingleton<IConfiguration>(configuration);
            services.AddLogging();
            services.AddHttpContextAccessor();
            services.AddScoped<CurrentUserService>();
            services.AddScoped<ICurrentUserService>(provider => provider.GetRequiredService<CurrentUserService>());
            services.AddDbContext<JobTrackerContext>((_, options) => options.UseSqlite(connection));
            services.AddIdentityCore<ApplicationUser>().AddRoles<IdentityRole>().AddEntityFrameworkStores<JobTrackerContext>();
            services.AddSingleton(TimeProvider.System);
            services.AddSingleton(new AppPaths(configuration, environment.Object));
            services.AddSingleton<AiPrivacyPolicy>();
            services.AddSingleton<AiOperationExecutionScope>();
            services.AddScoped<UserOperationStore>();
            services.AddScoped<AiUsageMeter>();
            services.AddScoped<AiOperationAdmission>();
            services.AddScoped<ICvProcessingQueue, CvProcessingQueue>();
            services.AddTransient<ProfileCvController>();
            services.AddSingleton<IAiOperationHandler, CvProcessingOperationHandler>();
            services.AddSingleton<AiOperationWorker>();
            services.AddSingleton(summarizer.Object);
            var provider = services.BuildServiceProvider();
            await using var scope = provider.CreateAsyncScope();
            await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().Database.EnsureCreatedAsync();
            fixture = new Fixture(connection, tempRoot, provider);
            return fixture;
        }

        public async Task SeedUserAsync(string? profileCvText = null)
        {
            await using var scope = Provider.CreateAsyncScope();
            var roles = scope.ServiceProvider.GetRequiredService<RoleManager<IdentityRole>>();
            Assert.True((await roles.CreateAsync(new IdentityRole("Premium"))).Succeeded);
            var users = scope.ServiceProvider.GetRequiredService<UserManager<ApplicationUser>>();
            var user = new ApplicationUser
            {
                Id = "user-1",
                UserName = "user-1@example.test",
                Email = "user-1@example.test",
                EmailConfirmed = true,
                AiEnabled = true,
                ProfileCvText = profileCvText,
            };
            Assert.True((await users.CreateAsync(user)).Succeeded);
            Assert.True((await users.AddToRoleAsync(user, "Premium")).Succeeded);
        }

        public async ValueTask DisposeAsync()
        {
            await Provider.DisposeAsync();
            await _connection.DisposeAsync();
            try { if (Directory.Exists(_tempRoot)) Directory.Delete(_tempRoot, recursive: true); } catch { }
        }
    }
}