feat(cv)!: queue durable processing
CV upload now returns 202 with an owner-scoped operation instead of holding the request through parsing. Existing review approval remains required. BREAKING CHANGE: profile-cv upload responses use the durable operation contract.
This commit is contained in:
@@ -0,0 +1,240 @@
|
||||
using System.Security.Claims;
|
||||
using System.Text;
|
||||
using JobTrackerApi.Controllers;
|
||||
using JobTrackerApi.Data;
|
||||
using JobTrackerApi.Models;
|
||||
using JobTrackerApi.Services;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace JobTrackerApi.Tests;
|
||||
|
||||
public sealed class CvProcessingOperationTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Upload_is_durable_idempotent_and_stops_at_review_gate()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
await fixture.SeedUserAsync();
|
||||
|
||||
CvProcessingOperationResponse first;
|
||||
CvProcessingOperationResponse duplicate;
|
||||
await using (var scope = fixture.Provider.CreateAsyncScope())
|
||||
{
|
||||
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
|
||||
var controller = CreateController(scope.ServiceProvider);
|
||||
first = Response(await controller.Upload(File()));
|
||||
duplicate = Response(await controller.Upload(File()));
|
||||
var runs = Assert.IsType<OkObjectResult>((await controller.GetRuns()).Result);
|
||||
Assert.Equal(first.Operation?.Id, Assert.Single(Assert.IsAssignableFrom<IEnumerable<CvExtractionRunListItem>>(runs.Value)).Operation?.Id);
|
||||
}
|
||||
|
||||
Assert.Equal(first.ExtractionRunId, duplicate.ExtractionRunId);
|
||||
Assert.Equal(first.Operation?.Id, duplicate.Operation?.Id);
|
||||
Assert.True(first.Created);
|
||||
Assert.False(duplicate.Created);
|
||||
|
||||
await using (var scope = fixture.Provider.CreateAsyncScope())
|
||||
{
|
||||
var db = scope.ServiceProvider.GetRequiredService<JobTrackerContext>();
|
||||
Assert.Single(await db.CvExtractionRuns.IgnoreQueryFilters().ToListAsync());
|
||||
Assert.Single(await db.CvUploadArtifacts.IgnoreQueryFilters().ToListAsync());
|
||||
var operation = Assert.Single(await db.UserOperations.IgnoreQueryFilters().ToListAsync());
|
||||
Assert.Equal(CvProcessingQueue.TaskType, operation.TaskType);
|
||||
Assert.Equal(CvProcessingQueue.SubjectType, operation.SubjectType);
|
||||
Assert.DoesNotContain("Ada", operation.SubjectId ?? string.Empty, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
|
||||
Assert.False(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
|
||||
|
||||
await using (var scope = fixture.Provider.CreateAsyncScope())
|
||||
{
|
||||
var db = scope.ServiceProvider.GetRequiredService<JobTrackerContext>();
|
||||
var run = await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync();
|
||||
var operation = await db.UserOperations.IgnoreQueryFilters().SingleAsync();
|
||||
var user = await db.Users.SingleAsync();
|
||||
Assert.Equal("pending_review", run.Status);
|
||||
Assert.Equal(OperationStatuses.Succeeded, operation.Status);
|
||||
Assert.Equal($"/api/profile-cv/runs/{run.Id}/diff", operation.ResultReference);
|
||||
Assert.Null(user.ProfileCvStructureJson);
|
||||
Assert.Null(user.CurrentCvExtractionRunId);
|
||||
Assert.Equal("operation_succeeded", (await db.UserNotifications.IgnoreQueryFilters().SingleAsync()).Kind);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Retryable_provider_failure_keeps_run_queued_and_records_provenance()
|
||||
{
|
||||
await using var fixture = await Fixture.CreateAsync();
|
||||
await fixture.SeedUserAsync(profileCvText: "# Ada Lovelace\n\n## Skills\nC#");
|
||||
fixture.GenerationFailure = new AiGenerationException(
|
||||
"provider_unavailable",
|
||||
"The local provider is unavailable.",
|
||||
retryable: true,
|
||||
provider: "ollama",
|
||||
model: "qwen-test",
|
||||
routeReason: "local_primary");
|
||||
|
||||
await using (var scope = fixture.Provider.CreateAsyncScope())
|
||||
{
|
||||
using var owner = scope.ServiceProvider.GetRequiredService<CurrentUserService>().UseBackgroundUser("user-1");
|
||||
Assert.IsType<AcceptedResult>(await CreateController(scope.ServiceProvider).Improve());
|
||||
}
|
||||
|
||||
Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
|
||||
|
||||
await using var verification = fixture.Provider.CreateAsyncScope();
|
||||
var db = verification.ServiceProvider.GetRequiredService<JobTrackerContext>();
|
||||
var operation = await db.UserOperations.IgnoreQueryFilters().SingleAsync();
|
||||
var run = await db.CvExtractionRuns.IgnoreQueryFilters().SingleAsync();
|
||||
Assert.Equal(OperationStatuses.WaitingForRetry, operation.Status);
|
||||
Assert.Equal("provider_unavailable", operation.FailureCategory);
|
||||
Assert.Equal("ollama", operation.Provider);
|
||||
Assert.Equal("qwen-test", operation.Model);
|
||||
Assert.Equal("local_primary", operation.ProgressStage);
|
||||
Assert.Equal("queued", run.Status);
|
||||
Assert.Null(run.CompletedAtUtc);
|
||||
}
|
||||
|
||||
private static ProfileCvController CreateController(IServiceProvider services)
|
||||
{
|
||||
var controller = services.GetRequiredService<ProfileCvController>();
|
||||
controller.ControllerContext = new ControllerContext
|
||||
{
|
||||
HttpContext = new DefaultHttpContext
|
||||
{
|
||||
User = new ClaimsPrincipal(new ClaimsIdentity(
|
||||
new[] { new Claim(ClaimTypes.NameIdentifier, "user-1") },
|
||||
"test")),
|
||||
},
|
||||
};
|
||||
return controller;
|
||||
}
|
||||
|
||||
private static CvProcessingOperationResponse Response(IActionResult result)
|
||||
=> Assert.IsType<CvProcessingOperationResponse>(Assert.IsType<AcceptedResult>(result).Value);
|
||||
|
||||
private static FormFile File()
|
||||
{
|
||||
const string text = "# Ada Lovelace\n\n## Professional Summary\nBuilt reliable analytical systems.\n\n## Skills\nC#\nSQL";
|
||||
var bytes = Encoding.UTF8.GetBytes(text);
|
||||
return new FormFile(new MemoryStream(bytes), 0, bytes.Length, "file", "synthetic-cv.md")
|
||||
{
|
||||
Headers = new HeaderDictionary(),
|
||||
ContentType = "text/markdown",
|
||||
};
|
||||
}
|
||||
|
||||
private sealed class Fixture : IAsyncDisposable
|
||||
{
|
||||
private readonly SqliteConnection _connection;
|
||||
private readonly string _tempRoot;
|
||||
public ServiceProvider Provider { get; }
|
||||
public AiGenerationException? GenerationFailure { get; set; }
|
||||
|
||||
private Fixture(SqliteConnection connection, string tempRoot, ServiceProvider provider)
|
||||
{
|
||||
_connection = connection;
|
||||
_tempRoot = tempRoot;
|
||||
Provider = provider;
|
||||
}
|
||||
|
||||
public static async Task<Fixture> CreateAsync()
|
||||
{
|
||||
var connection = new SqliteConnection("Data Source=:memory:");
|
||||
await connection.OpenAsync();
|
||||
var tempRoot = Path.Combine(Path.GetTempPath(), $"jobtracker-cv-operation-{Guid.NewGuid():N}");
|
||||
var configuration = new ConfigurationBuilder().AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Data:Root"] = tempRoot,
|
||||
["Data:CvArtifactsRoot"] = Path.Combine(tempRoot, "CvArtifacts"),
|
||||
["AiQueue:HeartbeatSeconds"] = "5",
|
||||
["Ai:ExternalProcessingEnabled"] = "false",
|
||||
}).Build();
|
||||
var environment = new Mock<IHostEnvironment>();
|
||||
environment.SetupGet(item => item.ContentRootPath).Returns(tempRoot);
|
||||
Fixture? fixture = null;
|
||||
var summarizer = new Mock<ISummarizerService>();
|
||||
summarizer.Setup(item => item.ExtractTextAsync(
|
||||
It.IsAny<Stream>(), It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new AiTextExtractionResult(
|
||||
"# Ada Lovelace\n\n## Professional Summary\nBuilt reliable analytical systems.\n\n## Skills\nC#\nSQL",
|
||||
false,
|
||||
"text/markdown",
|
||||
null,
|
||||
94,
|
||||
"synthetic-cv.md"));
|
||||
summarizer.Setup(item => item.SummarizeSectionAsync(
|
||||
It.Is<string>(instruction => instruction.Contains("structured JSON", StringComparison.Ordinal)),
|
||||
It.IsAny<string>(), 3200, 900))
|
||||
.ReturnsAsync("""
|
||||
{"version":"1","contact":{"fullName":"Ada Lovelace"},"summary":["Built reliable analytical systems."],"jobs":[],"education":[],"skills":["C#","SQL"],"languages":[],"interests":[],"otherSections":[]}
|
||||
""");
|
||||
summarizer.Setup(item => item.GenerateSectionWithMetadataAsync(
|
||||
It.IsAny<string>(), It.IsAny<string>(), It.IsAny<int>(), It.IsAny<int>(), It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(() => fixture!.GenerationFailure is null
|
||||
? new AiGenerationResult("# Ada Lovelace\n\n## Skills\nC#", "ollama", "qwen-test", RouteReason: "local_primary")
|
||||
: throw fixture.GenerationFailure);
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddSingleton<IConfiguration>(configuration);
|
||||
services.AddLogging();
|
||||
services.AddHttpContextAccessor();
|
||||
services.AddScoped<CurrentUserService>();
|
||||
services.AddScoped<ICurrentUserService>(provider => provider.GetRequiredService<CurrentUserService>());
|
||||
services.AddDbContext<JobTrackerContext>((_, options) => options.UseSqlite(connection));
|
||||
services.AddIdentityCore<ApplicationUser>().AddRoles<IdentityRole>().AddEntityFrameworkStores<JobTrackerContext>();
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddSingleton(new AppPaths(configuration, environment.Object));
|
||||
services.AddSingleton<AiPrivacyPolicy>();
|
||||
services.AddSingleton<AiOperationExecutionScope>();
|
||||
services.AddScoped<UserOperationStore>();
|
||||
services.AddScoped<AiOperationAdmission>();
|
||||
services.AddScoped<ICvProcessingQueue, CvProcessingQueue>();
|
||||
services.AddTransient<ProfileCvController>();
|
||||
services.AddSingleton<IAiOperationHandler, CvProcessingOperationHandler>();
|
||||
services.AddSingleton<AiOperationWorker>();
|
||||
services.AddSingleton(summarizer.Object);
|
||||
var provider = services.BuildServiceProvider();
|
||||
await using var scope = provider.CreateAsyncScope();
|
||||
await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().Database.EnsureCreatedAsync();
|
||||
fixture = new Fixture(connection, tempRoot, provider);
|
||||
return fixture;
|
||||
}
|
||||
|
||||
public async Task SeedUserAsync(string? profileCvText = null)
|
||||
{
|
||||
await using var scope = Provider.CreateAsyncScope();
|
||||
var roles = scope.ServiceProvider.GetRequiredService<RoleManager<IdentityRole>>();
|
||||
Assert.True((await roles.CreateAsync(new IdentityRole("Premium"))).Succeeded);
|
||||
var users = scope.ServiceProvider.GetRequiredService<UserManager<ApplicationUser>>();
|
||||
var user = new ApplicationUser
|
||||
{
|
||||
Id = "user-1",
|
||||
UserName = "user-1@example.test",
|
||||
Email = "user-1@example.test",
|
||||
EmailConfirmed = true,
|
||||
AiEnabled = true,
|
||||
ProfileCvText = profileCvText,
|
||||
};
|
||||
Assert.True((await users.CreateAsync(user)).Succeeded);
|
||||
Assert.True((await users.AddToRoleAsync(user, "Premium")).Succeeded);
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await Provider.DisposeAsync();
|
||||
await _connection.DisposeAsync();
|
||||
try { if (Directory.Exists(_tempRoot)) Directory.Delete(_tempRoot, recursive: true); } catch { }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -45,6 +45,7 @@ public sealed class ProfileCvControllerTests
|
||||
var user = new ApplicationUser { Id = "user-1" };
|
||||
var userManager = CreateUserManager();
|
||||
userManager.Setup(x => x.GetUserAsync(It.IsAny<ClaimsPrincipal>())).ReturnsAsync(user);
|
||||
ConfigureWorkerUser(userManager, user);
|
||||
userManager.Setup(x => x.UpdateAsync(user)).ReturnsAsync(IdentityResult.Success);
|
||||
var aiService = new Mock<ISummarizerService>();
|
||||
aiService
|
||||
@@ -76,9 +77,9 @@ public sealed class ProfileCvControllerTests
|
||||
ContentType = "text/markdown"
|
||||
};
|
||||
|
||||
var result = await controller.Upload(file);
|
||||
var result = await UploadAndProcessAsync(controller, db, file);
|
||||
|
||||
Assert.IsType<OkObjectResult>(result);
|
||||
Assert.IsType<AcceptedResult>(result);
|
||||
var artifact = await db.CvUploadArtifacts.SingleAsync();
|
||||
var run = await db.CvExtractionRuns.SingleAsync();
|
||||
Assert.Equal("user-1", artifact.OwnerUserId);
|
||||
@@ -235,10 +236,10 @@ public sealed class ProfileCvControllerTests
|
||||
userManager.Setup(x => x.FindByIdAsync(user.Id)).ReturnsAsync(user);
|
||||
userManager.Setup(x => x.GetRolesAsync(user)).ReturnsAsync(new[] { "Premium" });
|
||||
var aiService = new Mock<ISummarizerService>();
|
||||
aiService.Setup(x => x.SummarizeSectionAsync(
|
||||
aiService.Setup(x => x.GenerateSectionWithMetadataAsync(
|
||||
It.Is<string>(instruction => instruction.StartsWith("Rewrite this CV", StringComparison.Ordinal)),
|
||||
It.IsAny<string>(), 1800, 500))
|
||||
.ReturnsAsync(user.ProfileCvText);
|
||||
It.IsAny<string>(), 1800, 500, It.IsAny<CancellationToken>()))
|
||||
.ReturnsAsync(new AiGenerationResult(user.ProfileCvText));
|
||||
aiService.Setup(x => x.SummarizeSectionAsync(
|
||||
It.Is<string>(instruction => instruction.Contains("Extract this CV into structured JSON", StringComparison.Ordinal)),
|
||||
It.IsAny<string>(), 3200, 900))
|
||||
@@ -375,6 +376,7 @@ public sealed class ProfileCvControllerTests
|
||||
var user = new ApplicationUser { Id = "user-1" };
|
||||
var userManager = CreateUserManager();
|
||||
userManager.Setup(x => x.GetUserAsync(It.IsAny<ClaimsPrincipal>())).ReturnsAsync(user);
|
||||
ConfigureWorkerUser(userManager, user);
|
||||
userManager.Setup(x => x.UpdateAsync(user)).ReturnsAsync(IdentityResult.Success);
|
||||
var aiService = new Mock<ISummarizerService>();
|
||||
aiService
|
||||
@@ -398,9 +400,9 @@ public sealed class ProfileCvControllerTests
|
||||
ContentType = "application/pdf"
|
||||
};
|
||||
|
||||
var result = await controller.Upload(file);
|
||||
var result = await UploadAndProcessAsync(controller, db, file);
|
||||
|
||||
Assert.IsType<OkObjectResult>(result);
|
||||
Assert.IsType<AcceptedResult>(result);
|
||||
var savedRun = await db.CvExtractionRuns.SingleAsync();
|
||||
Assert.Equal(reconstructed, savedRun.NormalizedText);
|
||||
|
||||
@@ -425,6 +427,7 @@ public sealed class ProfileCvControllerTests
|
||||
var user = new ApplicationUser { Id = "user-1" };
|
||||
var userManager = CreateUserManager();
|
||||
userManager.Setup(x => x.GetUserAsync(It.IsAny<ClaimsPrincipal>())).ReturnsAsync(user);
|
||||
ConfigureWorkerUser(userManager, user);
|
||||
userManager.Setup(x => x.UpdateAsync(user)).ReturnsAsync(IdentityResult.Success);
|
||||
var aiService = new Mock<ISummarizerService>();
|
||||
aiService
|
||||
@@ -456,9 +459,9 @@ public sealed class ProfileCvControllerTests
|
||||
ContentType = "application/pdf"
|
||||
};
|
||||
|
||||
var result = await controller.Upload(file);
|
||||
var result = await UploadAndProcessAsync(controller, db, file);
|
||||
|
||||
Assert.IsType<OkObjectResult>(result);
|
||||
Assert.IsType<AcceptedResult>(result);
|
||||
normalizer.Verify(x => x.NormalizeAsync(It.Is<string>(text => text.Contains("Warwickshire County Council", StringComparison.Ordinal)), It.IsAny<CancellationToken>()), Times.Once);
|
||||
var savedRun = await db.CvExtractionRuns.SingleAsync();
|
||||
var structured = StructuredCvProfileJson.Deserialize(savedRun.StructuredProfileJson);
|
||||
@@ -475,6 +478,7 @@ public sealed class ProfileCvControllerTests
|
||||
var user = new ApplicationUser { Id = "user-1" };
|
||||
var userManager = CreateUserManager();
|
||||
userManager.Setup(x => x.GetUserAsync(It.IsAny<ClaimsPrincipal>())).ReturnsAsync(user);
|
||||
ConfigureWorkerUser(userManager, user);
|
||||
userManager.Setup(x => x.UpdateAsync(user)).ReturnsAsync(IdentityResult.Success);
|
||||
var aiService = new Mock<ISummarizerService>();
|
||||
aiService
|
||||
@@ -498,9 +502,9 @@ public sealed class ProfileCvControllerTests
|
||||
ContentType = "application/pdf"
|
||||
};
|
||||
|
||||
var result = await controller.Upload(file);
|
||||
var result = await UploadAndProcessAsync(controller, db, file);
|
||||
|
||||
Assert.IsType<OkObjectResult>(result);
|
||||
Assert.IsType<AcceptedResult>(result);
|
||||
var savedRun = await db.CvExtractionRuns.SingleAsync();
|
||||
var structured = StructuredCvProfileJson.Deserialize(savedRun.StructuredProfileJson);
|
||||
Assert.Equal("Connor Babbington", structured.Contact.FullName);
|
||||
@@ -1068,6 +1072,7 @@ public sealed class ProfileCvControllerTests
|
||||
var user = new ApplicationUser { Id = "user-1" };
|
||||
var userManager = CreateUserManager();
|
||||
userManager.Setup(x => x.GetUserAsync(It.IsAny<ClaimsPrincipal>())).ReturnsAsync(user);
|
||||
ConfigureWorkerUser(userManager, user);
|
||||
userManager.Setup(x => x.UpdateAsync(user)).ReturnsAsync(IdentityResult.Success);
|
||||
var aiService = new Mock<ISummarizerService>();
|
||||
aiService
|
||||
@@ -1098,9 +1103,9 @@ public sealed class ProfileCvControllerTests
|
||||
Headers = new HeaderDictionary(),
|
||||
ContentType = "text/markdown"
|
||||
};
|
||||
var result = await controller.Upload(file);
|
||||
var result = await UploadAndProcessAsync(controller, db, file);
|
||||
|
||||
Assert.IsType<OkObjectResult>(result);
|
||||
Assert.IsType<AcceptedResult>(result);
|
||||
var run = await db.CvExtractionRuns.SingleAsync();
|
||||
Assert.Contains("Built APIs", run.NormalizedText);
|
||||
Assert.Equal("Connor Babbington", StructuredCvProfileJson.Deserialize(run.StructuredProfileJson).Contact.FullName);
|
||||
@@ -1346,6 +1351,22 @@ public sealed class ProfileCvControllerTests
|
||||
return StructuredCvProfileJson.Normalize((StructuredCvProfile)result!);
|
||||
}
|
||||
|
||||
private static async Task<IActionResult> UploadAndProcessAsync(ProfileCvController controller, JobTrackerContext db, IFormFile file)
|
||||
{
|
||||
var accepted = Assert.IsType<AcceptedResult>(await controller.Upload(file));
|
||||
var run = await db.CvExtractionRuns.SingleAsync();
|
||||
var outcome = Assert.IsType<CvProcessingOutcome>(await controller.ProcessQueuedRunAsync(run.Id, CancellationToken.None));
|
||||
Assert.True(outcome.Succeeded, outcome.FailureMessage);
|
||||
return accepted;
|
||||
}
|
||||
|
||||
private static void ConfigureWorkerUser(Mock<UserManager<ApplicationUser>> userManager, ApplicationUser user)
|
||||
{
|
||||
user.AiEnabled = true;
|
||||
userManager.Setup(x => x.FindByIdAsync(user.Id)).ReturnsAsync(user);
|
||||
userManager.Setup(x => x.GetRolesAsync(user)).ReturnsAsync(new[] { "Premium" });
|
||||
}
|
||||
|
||||
private static ProfileCvController CreateController(UserManager<ApplicationUser> userManager, ISummarizerService aiService, JobTrackerContext db, AppPaths paths, ICvAiClassifier? cvAiClassifier = null, ICvAiNormalizer? cvAiNormalizer = null)
|
||||
{
|
||||
return new ProfileCvController(userManager, aiService, db, paths, null, cvAiClassifier ?? NoOpCvAiClassifier.Instance, cvAiNormalizer ?? NoOpCvAiNormalizer.Instance)
|
||||
|
||||
@@ -293,6 +293,9 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
|
||||
private async Task<CvExtractionRun> CreateQueuedRunAsync(string ownerUserId, int? artifactId, string trigger, CancellationToken cancellationToken)
|
||||
{
|
||||
var active = await FindActiveRunAsync(ownerUserId, trigger, artifactId, null, cancellationToken);
|
||||
if (active is not null) return active;
|
||||
|
||||
var run = new CvExtractionRun
|
||||
{
|
||||
OwnerUserId = ownerUserId,
|
||||
@@ -309,14 +312,90 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
return run;
|
||||
}
|
||||
|
||||
// Invoked by CvProcessingHostedService (this controller is also registered as a
|
||||
private async Task<CvExtractionRun?> FindActiveRunAsync(
|
||||
string ownerUserId,
|
||||
string trigger,
|
||||
int? artifactId,
|
||||
string? artifactSha256,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var activeStatuses = new[]
|
||||
{
|
||||
OperationStatuses.Queued,
|
||||
OperationStatuses.Running,
|
||||
OperationStatuses.WaitingForRetry,
|
||||
OperationStatuses.WaitingForExternalFallback,
|
||||
};
|
||||
var subjectIds = await _db.UserOperations.AsNoTracking()
|
||||
.Where(operation => operation.TaskType == CvProcessingQueue.TaskType && activeStatuses.Contains(operation.Status))
|
||||
.Select(operation => operation.SubjectId)
|
||||
.ToListAsync(cancellationToken);
|
||||
var runIds = subjectIds
|
||||
.Select(value => int.TryParse(value, out var id) ? id : 0)
|
||||
.Where(id => id > 0)
|
||||
.ToList();
|
||||
if (runIds.Count == 0) return null;
|
||||
|
||||
var candidates = await _db.CvExtractionRuns
|
||||
.Include(run => run.Artifact)
|
||||
.Where(run => run.OwnerUserId == ownerUserId && run.Trigger == trigger && runIds.Contains(run.Id))
|
||||
.ToListAsync(cancellationToken);
|
||||
return candidates
|
||||
.Where(run => artifactSha256 is not null
|
||||
? string.Equals(run.Artifact?.Sha256, artifactSha256, StringComparison.OrdinalIgnoreCase)
|
||||
: run.ArtifactId == artifactId)
|
||||
.MaxBy(run => run.StartedAtUtc);
|
||||
}
|
||||
|
||||
private async Task<IActionResult> EnqueueRunAsync(CvExtractionRun run, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var admission = await _cvProcessingQueue.EnqueueAsync(run.Id, cancellationToken);
|
||||
return Accepted(
|
||||
admission?.StatusUrl,
|
||||
new CvProcessingOperationResponse(
|
||||
true,
|
||||
run.Id,
|
||||
run.Status,
|
||||
admission is null ? null : OperationDto.From(admission.Operation),
|
||||
admission?.StatusUrl,
|
||||
admission?.Created ?? false));
|
||||
}
|
||||
catch (AiOperationAdmissionException exception)
|
||||
{
|
||||
run.Status = "failed";
|
||||
run.ErrorMessage = exception.Message;
|
||||
run.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
await _db.SaveChangesAsync(cancellationToken);
|
||||
if (exception.RetryAfterSeconds is int seconds) Response.Headers.RetryAfter = seconds.ToString();
|
||||
return StatusCode(exception.StatusCode, new { code = exception.Code, message = exception.Message });
|
||||
}
|
||||
}
|
||||
|
||||
private void TryDeleteCvArtifactFile(string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (System.IO.File.Exists(path)) System.IO.File.Delete(path);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(exception, "Could not remove duplicate CV upload artifact {ArtifactPath}", path);
|
||||
}
|
||||
}
|
||||
|
||||
// Invoked by CvProcessingOperationHandler (this controller is also registered as a
|
||||
// transient service). NonAction keeps it off the HTTP surface: without it the
|
||||
// controller-level [Route] exposes it as an any-verb endpoint.
|
||||
[NonAction]
|
||||
public async Task ProcessQueuedRunAsync(int runId, CancellationToken cancellationToken)
|
||||
public async Task<CvProcessingOutcome?> ProcessQueuedRunAsync(int runId, CancellationToken cancellationToken)
|
||||
{
|
||||
var run = await _db.CvExtractionRuns.IgnoreQueryFilters().FirstOrDefaultAsync(x => x.Id == runId, cancellationToken);
|
||||
if (run is null) return;
|
||||
var ownerUserId = _db.CurrentUserId;
|
||||
var run = ownerUserId is null
|
||||
? await _db.CvExtractionRuns.IgnoreQueryFilters().FirstOrDefaultAsync(x => x.Id == runId, cancellationToken)
|
||||
: await _db.CvExtractionRuns.FirstOrDefaultAsync(x => x.Id == runId && x.OwnerUserId == ownerUserId, cancellationToken);
|
||||
if (run is null) return null;
|
||||
var user = await _users.FindByIdAsync(run.OwnerUserId);
|
||||
if (user is null)
|
||||
{
|
||||
@@ -324,7 +403,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
run.ErrorMessage = "CV processing user was not found.";
|
||||
run.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
await _db.SaveChangesAsync(cancellationToken);
|
||||
return;
|
||||
return new CvProcessingOutcome(false, "cv_user_not_found", run.ErrorMessage);
|
||||
}
|
||||
|
||||
if (!user.AiEnabled || !AccountPlans.ForRoles(await _users.GetRolesAsync(user)).Ai)
|
||||
@@ -335,7 +414,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
: "AI is disabled in your privacy settings.";
|
||||
run.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
await _db.SaveChangesAsync(cancellationToken);
|
||||
return;
|
||||
return new CvProcessingOutcome(false, "entitlement_changed", run.ErrorMessage);
|
||||
}
|
||||
|
||||
run.Status = "running";
|
||||
@@ -344,16 +423,19 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
|
||||
try
|
||||
{
|
||||
AiGenerationResult? generation = null;
|
||||
switch (run.Trigger)
|
||||
{
|
||||
case "rebuild":
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(user.ProfileCvText)) throw new InvalidOperationException("Add or import CV text before rebuilding it.");
|
||||
var rebuilt = await _aiService.SummarizeSectionAsync(
|
||||
generation = await _aiService.GenerateSectionWithMetadataAsync(
|
||||
"Rewrite this CV into a stronger master CV with clear sections such as Professional Summary, Core Skills, Experience Highlights, and Selected Achievements. Preserve only factual claims, avoid inventing employers or metrics, and make the output clean and ready for tailoring to job applications. Return only the rebuilt CV text.",
|
||||
user.ProfileCvText,
|
||||
2200,
|
||||
700);
|
||||
700,
|
||||
cancellationToken);
|
||||
var rebuilt = generation?.Text;
|
||||
if (string.IsNullOrWhiteSpace(rebuilt)) throw new InvalidOperationException("The AI service could not rebuild your CV text right now.");
|
||||
|
||||
var normalizedText = rebuilt.Trim();
|
||||
@@ -364,11 +446,13 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
case "improve":
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(user.ProfileCvText)) throw new InvalidOperationException("Add or import CV text before improving it.");
|
||||
var improved = await _aiService.SummarizeSectionAsync(
|
||||
generation = await _aiService.GenerateSectionWithMetadataAsync(
|
||||
"Rewrite this CV into a cleaner, better-structured master CV profile. Preserve factual claims, employers, skills, and measurable results. Improve clarity, tighten wording, use strong bullet-style phrasing, and keep it ready for further tailoring to specific roles. Return only the improved CV text.",
|
||||
user.ProfileCvText,
|
||||
1800,
|
||||
500);
|
||||
500,
|
||||
cancellationToken);
|
||||
var improved = generation?.Text;
|
||||
if (string.IsNullOrWhiteSpace(improved)) throw new InvalidOperationException("The AI service could not improve your CV text right now.");
|
||||
|
||||
var normalizedText = improved.Trim();
|
||||
@@ -376,6 +460,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
await CompleteQueuedRunForReviewAsync(run, normalizedText, normalizedText, structuredCv, cancellationToken);
|
||||
break;
|
||||
}
|
||||
case "upload":
|
||||
case "reprocess":
|
||||
{
|
||||
var artifact = await _db.CvUploadArtifacts.IgnoreQueryFilters().AsNoTracking().FirstOrDefaultAsync(x => x.Id == run.ArtifactId && x.OwnerUserId == user.Id, cancellationToken);
|
||||
@@ -401,16 +486,42 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
}
|
||||
|
||||
await SendRunCompletionEmailAsync(user, run, true, cancellationToken);
|
||||
return new CvProcessingOutcome(
|
||||
true,
|
||||
Provider: generation?.Provider,
|
||||
Model: generation?.Model,
|
||||
RouteReason: generation?.RouteReason);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
run.Status = "queued";
|
||||
run.ErrorMessage = "CV processing was interrupted before completion.";
|
||||
run.CompletedAtUtc = null;
|
||||
await _db.SaveChangesAsync(CancellationToken.None);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
run.Status = "failed";
|
||||
var generationFailure = ex as AiGenerationException;
|
||||
var retryable = generationFailure?.Retryable == true;
|
||||
run.Status = retryable ? "queued" : "failed";
|
||||
run.ErrorMessage = ex.Message;
|
||||
run.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
run.CompletedAtUtc = retryable ? null : DateTimeOffset.UtcNow;
|
||||
await _db.SaveChangesAsync(cancellationToken);
|
||||
await PruneExtractionRunsAsync(user.Id, cancellationToken);
|
||||
await SendRunCompletionEmailAsync(user, run, false, cancellationToken);
|
||||
if (!retryable)
|
||||
{
|
||||
await PruneExtractionRunsAsync(user.Id, cancellationToken);
|
||||
await SendRunCompletionEmailAsync(user, run, false, cancellationToken);
|
||||
}
|
||||
_logger.LogWarning(ex, "CV processing run {RunId} failed for user {UserId}", run.Id, user.Id);
|
||||
return new CvProcessingOutcome(
|
||||
false,
|
||||
generationFailure?.Category ?? "cv_processing_failed",
|
||||
ex.Message,
|
||||
retryable,
|
||||
generationFailure?.Provider,
|
||||
generationFailure?.Model,
|
||||
generationFailure?.RouteReason);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -152,6 +152,12 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
}
|
||||
|
||||
var artifact = await SaveUploadArtifactAsync(user, file, HttpContext.RequestAborted);
|
||||
var activeRun = await FindActiveRunAsync(user.Id, "upload", null, artifact.Sha256, HttpContext.RequestAborted);
|
||||
if (activeRun is not null)
|
||||
{
|
||||
TryDeleteCvArtifactFile(artifact.StoragePath);
|
||||
return await EnqueueRunAsync(activeRun, HttpContext.RequestAborted);
|
||||
}
|
||||
_db.CvUploadArtifacts.Add(artifact);
|
||||
await _db.SaveChangesAsync(HttpContext.RequestAborted);
|
||||
|
||||
@@ -163,42 +169,12 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
ParserVersion = ParserVersion,
|
||||
NormalizerVersion = NormalizerVersion,
|
||||
LlmPromptVersion = LlmPromptVersion,
|
||||
Status = "running",
|
||||
Status = "queued",
|
||||
StartedAtUtc = DateTimeOffset.UtcNow,
|
||||
};
|
||||
_db.CvExtractionRuns.Add(run);
|
||||
await _db.SaveChangesAsync(HttpContext.RequestAborted);
|
||||
|
||||
try
|
||||
{
|
||||
var result = await ExtractStructuredCvFromFileAsync(file, extension, HttpContext.RequestAborted);
|
||||
run.RawExtractedText = result.RawText;
|
||||
run.NormalizedText = result.NormalizedText;
|
||||
run.StructuredProfileJson = StructuredCvProfileJson.Serialize(result.StructuredCv);
|
||||
run.Status = "pending_review";
|
||||
run.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
await _db.SaveChangesAsync(HttpContext.RequestAborted);
|
||||
await PruneExtractionRunsAsync(user.Id, HttpContext.RequestAborted);
|
||||
|
||||
return Ok(new
|
||||
{
|
||||
imported = false,
|
||||
pendingReview = true,
|
||||
characters = result.NormalizedText.Length,
|
||||
artifactId = artifact.Id,
|
||||
extractionRunId = run.Id,
|
||||
status = run.Status,
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
run.Status = "failed";
|
||||
run.ErrorMessage = ex.Message;
|
||||
run.CompletedAtUtc = DateTimeOffset.UtcNow;
|
||||
await _db.SaveChangesAsync(HttpContext.RequestAborted);
|
||||
await PruneExtractionRunsAsync(user.Id, HttpContext.RequestAborted);
|
||||
throw;
|
||||
}
|
||||
return await EnqueueRunAsync(run, HttpContext.RequestAborted);
|
||||
}
|
||||
|
||||
[HttpGet("runs")]
|
||||
@@ -221,11 +197,23 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
x.ParserVersion,
|
||||
x.NormalizerVersion,
|
||||
x.LlmPromptVersion,
|
||||
x.ErrorMessage));
|
||||
x.ErrorMessage,
|
||||
null));
|
||||
var runs = _db.Database.IsSqlite()
|
||||
? (await runsQuery.ToListAsync(HttpContext.RequestAborted)).OrderByDescending(x => x.StartedAtUtc).Take(10).ToList()
|
||||
: await runsQuery.OrderByDescending(x => x.StartedAtUtc).Take(10).ToListAsync(HttpContext.RequestAborted);
|
||||
|
||||
var runIds = runs.Select(run => run.Id.ToString(System.Globalization.CultureInfo.InvariantCulture)).ToList();
|
||||
var operations = await _db.UserOperations.AsNoTracking()
|
||||
.Where(operation => operation.TaskType == CvProcessingQueue.TaskType && operation.SubjectId != null && runIds.Contains(operation.SubjectId))
|
||||
.ToListAsync(HttpContext.RequestAborted);
|
||||
var latestOperations = operations
|
||||
.GroupBy(operation => operation.SubjectId!, StringComparer.Ordinal)
|
||||
.ToDictionary(group => group.Key, group => group.MaxBy(operation => operation.CreatedAtUtc)!);
|
||||
runs = runs.Select(run => latestOperations.TryGetValue(run.Id.ToString(System.Globalization.CultureInfo.InvariantCulture), out var operation)
|
||||
? run with { Operation = OperationDto.From(operation) }
|
||||
: run).ToList();
|
||||
|
||||
return Ok(runs);
|
||||
}
|
||||
|
||||
@@ -313,8 +301,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
}
|
||||
|
||||
var run = await CreateQueuedRunAsync(user.Id, artifact.Id, "reprocess", HttpContext.RequestAborted);
|
||||
await _cvProcessingQueue.EnqueueAsync(run.Id, HttpContext.RequestAborted);
|
||||
return Accepted(new { queued = true, extractionRunId = run.Id, status = run.Status });
|
||||
return await EnqueueRunAsync(run, HttpContext.RequestAborted);
|
||||
}
|
||||
|
||||
[HttpPost("rebuild")]
|
||||
@@ -326,8 +313,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
if (string.IsNullOrWhiteSpace(user.ProfileCvText)) return BadRequest("Add or import CV text before rebuilding it.");
|
||||
|
||||
var run = await CreateQueuedRunAsync(user.Id, user.CurrentCvUploadArtifactId, "rebuild", HttpContext.RequestAborted);
|
||||
await _cvProcessingQueue.EnqueueAsync(run.Id, HttpContext.RequestAborted);
|
||||
return Accepted(new { queued = true, extractionRunId = run.Id, status = run.Status });
|
||||
return await EnqueueRunAsync(run, HttpContext.RequestAborted);
|
||||
}
|
||||
|
||||
[HttpPost("rewrite-section")]
|
||||
@@ -527,8 +513,7 @@ public sealed partial class ProfileCvController : ControllerBase
|
||||
if (string.IsNullOrWhiteSpace(user.ProfileCvText)) return BadRequest("Add or import CV text before improving it.");
|
||||
|
||||
var run = await CreateQueuedRunAsync(user.Id, user.CurrentCvUploadArtifactId, "improve", HttpContext.RequestAborted);
|
||||
await _cvProcessingQueue.EnqueueAsync(run.Id, HttpContext.RequestAborted);
|
||||
return Accepted(new { queued = true, extractionRunId = run.Id, status = run.Status });
|
||||
return await EnqueueRunAsync(run, HttpContext.RequestAborted);
|
||||
}
|
||||
|
||||
private static string BuildRewriteSourceText(string? sectionName, string? sourceText, StructuredCvProfile structuredCv)
|
||||
|
||||
@@ -17,4 +17,12 @@ public sealed record CvExtractionRunListItem(
|
||||
string ParserVersion,
|
||||
string NormalizerVersion,
|
||||
string LlmPromptVersion,
|
||||
string? ErrorMessage);
|
||||
string? ErrorMessage,
|
||||
OperationDto? Operation);
|
||||
public sealed record CvProcessingOperationResponse(
|
||||
bool Queued,
|
||||
int ExtractionRunId,
|
||||
string Status,
|
||||
OperationDto? Operation,
|
||||
string? StatusUrl,
|
||||
bool Created);
|
||||
|
||||
@@ -48,11 +48,12 @@ builder.Services.AddScoped<UserOperationStore>();
|
||||
builder.Services.AddScoped<AiOperationAdmission>();
|
||||
builder.Services.AddScoped<StrategySnapshotService>();
|
||||
builder.Services.AddSingleton<IAiOperationHandler, StrategySnapshotOperationHandler>();
|
||||
builder.Services.AddSingleton<IAiOperationHandler, CvProcessingOperationHandler>();
|
||||
builder.Services.AddSingleton<AiOperationWorker>();
|
||||
builder.Services.AddScoped<UserNotificationStore>();
|
||||
builder.Services.AddScoped<IEmailSettingsResolver, EmailSettingsResolver>();
|
||||
builder.Services.AddScoped<IAppEmailSender, SmtpEmailSender>();
|
||||
builder.Services.AddSingleton<ICvProcessingQueue, CvProcessingQueue>();
|
||||
builder.Services.AddScoped<ICvProcessingQueue, CvProcessingQueue>();
|
||||
builder.Services.AddTransient<ProfileCvController>();
|
||||
builder.Services.AddSingleton<ICvTemplateRenderer, CvTemplateRenderer>();
|
||||
builder.Services.AddSingleton<IThemedCvRenderer, ThemedCvRenderer>();
|
||||
@@ -165,7 +166,6 @@ builder.Services.AddHostedService<FollowUpReminderHostedService>();
|
||||
builder.Services.AddHostedService<DailyExportHostedService>();
|
||||
builder.Services.AddHostedService<JobEnrichmentHostedService>();
|
||||
builder.Services.AddHostedService<SummarizerProbeHostedService>();
|
||||
builder.Services.AddHostedService<CvProcessingHostedService>();
|
||||
builder.Services.AddHostedService<AiOperationHostedService>();
|
||||
|
||||
builder.Services.AddHttpClient("jobimport")
|
||||
|
||||
@@ -1,97 +1,129 @@
|
||||
using System.Threading.Channels;
|
||||
using JobTrackerApi.Controllers;
|
||||
using JobTrackerApi.Data;
|
||||
using JobTrackerApi.Models;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace JobTrackerApi.Services;
|
||||
|
||||
public sealed record CvProcessingOutcome(
|
||||
bool Succeeded,
|
||||
string? FailureCategory = null,
|
||||
string? FailureMessage = null,
|
||||
bool Retryable = false,
|
||||
string? Provider = null,
|
||||
string? Model = null,
|
||||
string? RouteReason = null);
|
||||
|
||||
public interface ICvProcessingQueue
|
||||
{
|
||||
ValueTask EnqueueAsync(int runId, CancellationToken cancellationToken = default);
|
||||
IAsyncEnumerable<int> DequeueAllAsync(CancellationToken cancellationToken);
|
||||
Task<AiOperationAdmissionResult?> EnqueueAsync(int runId, CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
public sealed class CvProcessingQueue : ICvProcessingQueue
|
||||
/// <summary>
|
||||
/// Compatibility name for the CV producer bridge. Durable scheduling and execution are owned by
|
||||
/// the shared AI operation queue; this type does not keep an in-memory CV queue.
|
||||
/// </summary>
|
||||
public sealed class CvProcessingQueue(AiOperationAdmission admission) : ICvProcessingQueue
|
||||
{
|
||||
private readonly Channel<int> _channel = Channel.CreateUnbounded<int>(new UnboundedChannelOptions
|
||||
{
|
||||
SingleReader = true,
|
||||
SingleWriter = false,
|
||||
});
|
||||
public const string TaskType = "cv.process";
|
||||
public const string SubjectType = "cv_extraction_run";
|
||||
|
||||
public ValueTask EnqueueAsync(int runId, CancellationToken cancellationToken = default)
|
||||
=> _channel.Writer.WriteAsync(runId, cancellationToken);
|
||||
|
||||
public IAsyncEnumerable<int> DequeueAllAsync(CancellationToken cancellationToken)
|
||||
=> _channel.Reader.ReadAllAsync(cancellationToken);
|
||||
public async Task<AiOperationAdmissionResult?> EnqueueAsync(int runId, CancellationToken cancellationToken = default)
|
||||
=> await admission.EnqueueAsync(
|
||||
TaskType,
|
||||
$"run:{runId}",
|
||||
SubjectType,
|
||||
runId.ToString(System.Globalization.CultureInfo.InvariantCulture),
|
||||
AiOperationPriorities.UserVisible,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
public sealed class NoOpCvProcessingQueue : ICvProcessingQueue
|
||||
{
|
||||
public static readonly NoOpCvProcessingQueue Instance = new();
|
||||
public ValueTask EnqueueAsync(int runId, CancellationToken cancellationToken = default) => ValueTask.CompletedTask;
|
||||
public async IAsyncEnumerable<int> DequeueAllAsync([System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
public Task<AiOperationAdmissionResult?> EnqueueAsync(int runId, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult<AiOperationAdmissionResult?>(null);
|
||||
}
|
||||
|
||||
public sealed class CvProcessingHostedService : BackgroundService
|
||||
public sealed class CvProcessingOperationHandler : IAiOperationHandler
|
||||
{
|
||||
private readonly IServiceScopeFactory _scopeFactory;
|
||||
private readonly ICvProcessingQueue _queue;
|
||||
private readonly ILogger<CvProcessingHostedService> _logger;
|
||||
public string TaskType => CvProcessingQueue.TaskType;
|
||||
|
||||
public CvProcessingHostedService(IServiceScopeFactory scopeFactory, ICvProcessingQueue queue, ILogger<CvProcessingHostedService> logger)
|
||||
public async Task<AiOperationExecutionResult> ExecuteAsync(
|
||||
AiOperationExecutionContext context,
|
||||
IServiceProvider services,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
_scopeFactory = scopeFactory;
|
||||
_queue = queue;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await ProcessInterruptedRunsAsync(stoppingToken);
|
||||
|
||||
await foreach (var runId in _queue.DequeueAllAsync(stoppingToken))
|
||||
if (!string.Equals(context.Lease.SubjectType, CvProcessingQueue.SubjectType, StringComparison.Ordinal) ||
|
||||
!int.TryParse(context.Lease.SubjectId, out var runId) || runId <= 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessRunAsync(runId, stoppingToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Unhandled CV processing worker failure for run {RunId}", runId);
|
||||
}
|
||||
throw new AiOperationFailure("invalid_cv_run", "The CV processing operation has an invalid run reference.", retryable: false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ProcessInterruptedRunsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await using var scope = _scopeFactory.CreateAsyncScope();
|
||||
var db = scope.ServiceProvider.GetRequiredService<JobTrackerContext>();
|
||||
var interruptedRuns = await db.CvExtractionRuns.IgnoreQueryFilters()
|
||||
.Where(x => x.Status == "queued" || x.Status == "running")
|
||||
.Select(x => new { x.Id, x.StartedAtUtc })
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
// ponytail: single-instance recovery; use row leasing if multiple workers are ever deployed.
|
||||
// SQLite cannot ORDER BY DateTimeOffset, so the small interrupted-work set is ordered locally.
|
||||
foreach (var run in interruptedRuns.OrderBy(x => x.StartedAtUtc))
|
||||
CvProcessingOutcome? outcome;
|
||||
try
|
||||
{
|
||||
await ProcessRunAsync(run.Id, cancellationToken);
|
||||
outcome = await services.GetRequiredService<ProfileCvController>()
|
||||
.ProcessQueuedRunAsync(runId, cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
var operation = await services.GetRequiredService<UserOperationStore>()
|
||||
.GetAsync(context.Lease.OperationId, CancellationToken.None);
|
||||
await SetRunStatusAsync(
|
||||
services,
|
||||
runId,
|
||||
operation?.CancellationRequestedAtUtc is null ? "queued" : "cancelled",
|
||||
operation?.CancellationRequestedAtUtc is null ? "CV processing timed out and may be retried." : "CV processing was cancelled.");
|
||||
throw;
|
||||
}
|
||||
if (outcome is null)
|
||||
throw new AiOperationFailure("cv_run_not_found", "The CV processing run is no longer available.", retryable: false);
|
||||
if (!outcome.Succeeded)
|
||||
{
|
||||
if (outcome.Retryable)
|
||||
{
|
||||
var operation = await services.GetRequiredService<UserOperationStore>()
|
||||
.GetAsync(context.Lease.OperationId, cancellationToken);
|
||||
var canRetry = operation is not null && operation.AttemptCount < operation.MaxAttempts &&
|
||||
(operation.DeadlineAtUtc is null || operation.DeadlineAtUtc > DateTime.UtcNow);
|
||||
if (!canRetry)
|
||||
await SetRunStatusAsync(services, runId, "failed", outcome.FailureMessage ?? "CV processing failed.");
|
||||
}
|
||||
|
||||
if (outcome.Provider is not null || outcome.Model is not null || outcome.RouteReason is not null)
|
||||
{
|
||||
throw new AiGenerationException(
|
||||
outcome.FailureCategory ?? "cv_processing_failed",
|
||||
outcome.FailureMessage ?? "CV processing failed.",
|
||||
outcome.Retryable,
|
||||
outcome.Provider,
|
||||
outcome.Model,
|
||||
outcome.RouteReason);
|
||||
}
|
||||
|
||||
throw new AiOperationFailure(
|
||||
outcome.FailureCategory ?? "cv_processing_failed",
|
||||
outcome.FailureMessage ?? "CV processing failed.",
|
||||
outcome.Retryable);
|
||||
}
|
||||
|
||||
return new AiOperationExecutionResult(
|
||||
$"/api/profile-cv/runs/{runId}/diff",
|
||||
outcome.Provider,
|
||||
outcome.Model,
|
||||
outcome.RouteReason ?? "cv_pipeline");
|
||||
}
|
||||
|
||||
private async Task ProcessRunAsync(int runId, CancellationToken cancellationToken)
|
||||
private static Task<int> SetRunStatusAsync(IServiceProvider services, int runId, string status, string message)
|
||||
{
|
||||
await using var scope = _scopeFactory.CreateAsyncScope();
|
||||
var controller = scope.ServiceProvider.GetRequiredService<ProfileCvController>();
|
||||
await controller.ProcessQueuedRunAsync(runId, cancellationToken);
|
||||
var completedAtUtc = status == "failed" || status == "cancelled" ? DateTimeOffset.UtcNow : (DateTimeOffset?)null;
|
||||
return services.GetRequiredService<JobTrackerContext>().CvExtractionRuns
|
||||
.Where(run => run.Id == runId)
|
||||
.ExecuteUpdateAsync(setters => setters
|
||||
.SetProperty(run => run.Status, status)
|
||||
.SetProperty(run => run.ErrorMessage, message)
|
||||
.SetProperty(run => run.CompletedAtUtc, completedAtUtc),
|
||||
CancellationToken.None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -284,6 +284,37 @@ test('profile page can reprocess from stored artifact history', async () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('profile page shows durable CV operation state and retries a failed run', async () => {
|
||||
extractionRunsResponse = [{
|
||||
id: 14,
|
||||
trigger: 'upload',
|
||||
status: 'queued',
|
||||
artifactFileName: 'synthetic-cv.md',
|
||||
startedAtUtc: '2026-03-28T12:00:00Z',
|
||||
parserVersion: 'm005-s01',
|
||||
normalizerVersion: 'm005-s01',
|
||||
llmPromptVersion: 'm005-s01',
|
||||
operation: {
|
||||
id: '00000000-0000-0000-0000-000000000014',
|
||||
taskType: 'cv.process',
|
||||
status: 'failed',
|
||||
subjectType: 'cv_extraction_run',
|
||||
createdAtUtc: '2026-03-28T12:00:00Z',
|
||||
failureCategory: 'provider_unavailable',
|
||||
canCancel: false,
|
||||
canRetry: true,
|
||||
},
|
||||
}];
|
||||
renderPage();
|
||||
|
||||
expect(await screen.findByText('failed')).toBeInTheDocument();
|
||||
fireEvent.click(screen.getByRole('button', { name: /retry processing/i }));
|
||||
|
||||
await waitFor(() => {
|
||||
expect(mockedApi.post).toHaveBeenCalledWith('/operations/00000000-0000-0000-0000-000000000014/retry');
|
||||
});
|
||||
});
|
||||
|
||||
test('profile page keeps raw extraction collapsed until expanded', async () => {
|
||||
renderPage();
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ import {
|
||||
import { useToast } from "../toast";
|
||||
import { useI18n } from "../i18n/I18nProvider";
|
||||
import { useAccountPlan } from "../accountPlan";
|
||||
import type { UserOperation } from "../types";
|
||||
import {
|
||||
emptyStructuredCv,
|
||||
getStructuredCvFieldMetadata,
|
||||
@@ -52,6 +53,7 @@ type ExtractionRun = {
|
||||
normalizerVersion: string;
|
||||
llmPromptVersion: string;
|
||||
errorMessage?: string;
|
||||
operation?: UserOperation | null;
|
||||
};
|
||||
|
||||
type CvImportDiff = {
|
||||
@@ -74,8 +76,26 @@ type QueuedCvRunResponse = {
|
||||
queued: boolean;
|
||||
extractionRunId: number;
|
||||
status: string;
|
||||
operation?: UserOperation | null;
|
||||
statusUrl?: string | null;
|
||||
created: boolean;
|
||||
};
|
||||
|
||||
const activeOperationStatuses = new Set(["queued", "running", "waiting_for_retry", "waiting_for_external_fallback"]);
|
||||
const activeRunLabels = new Set(["queued", "running", "processing locally", "waiting to retry", "waiting for approved fallback"]);
|
||||
|
||||
function cvRunStatus(run: ExtractionRun) {
|
||||
const operation = run.operation;
|
||||
if (!operation || operation.status === "succeeded") return run.status;
|
||||
if (operation.cancellationRequestedAtUtc) return "cancellation requested";
|
||||
switch (operation.status) {
|
||||
case "running": return "processing locally";
|
||||
case "waiting_for_retry": return "waiting to retry";
|
||||
case "waiting_for_external_fallback": return "waiting for approved fallback";
|
||||
default: return operation.status;
|
||||
}
|
||||
}
|
||||
|
||||
type MeResponse = {
|
||||
provider?: "local" | "google" | "external";
|
||||
id?: string;
|
||||
@@ -208,7 +228,9 @@ export default function CareerProfilePage() {
|
||||
}, [loadProfile, loadVersions]);
|
||||
|
||||
useEffect(() => {
|
||||
const activeRuns = extractionRuns.filter((run) => run.status === "queued" || run.status === "running");
|
||||
const activeRuns = extractionRuns.filter((run) => run.operation
|
||||
? activeOperationStatuses.has(run.operation.status)
|
||||
: run.status === "queued" || run.status === "running");
|
||||
if (activeRuns.length === 0) return;
|
||||
|
||||
const timer = window.setInterval(() => {
|
||||
@@ -232,14 +254,15 @@ export default function CareerProfilePage() {
|
||||
useEffect(() => {
|
||||
const previous = runStatusRef.current;
|
||||
for (const run of extractionRuns) {
|
||||
const status = cvRunStatus(run);
|
||||
const prior = previous[run.id];
|
||||
if ((prior === "queued" || prior === "running") && run.status === "pending_review") {
|
||||
if (activeRunLabels.has(prior) && status === "pending_review") {
|
||||
toast(`CV ${run.trigger} is ready to review.`, "info");
|
||||
}
|
||||
if ((prior === "queued" || prior === "running") && run.status === "failed") {
|
||||
if (activeRunLabels.has(prior) && status === "failed") {
|
||||
toast(run.errorMessage || `CV ${run.trigger} failed.`, "error");
|
||||
}
|
||||
previous[run.id] = run.status;
|
||||
previous[run.id] = status;
|
||||
}
|
||||
}, [extractionRuns, toast]);
|
||||
|
||||
@@ -392,9 +415,9 @@ export default function CareerProfilePage() {
|
||||
formData.append("file", file);
|
||||
setUploadingCv(true);
|
||||
try {
|
||||
await api.post<QueuedCvRunResponse>("/profile-cv/upload", formData, { headers: { "Content-Type": "multipart/form-data" } });
|
||||
const res = await api.post<QueuedCvRunResponse>("/profile-cv/upload", formData, { headers: { "Content-Type": "multipart/form-data" } });
|
||||
await loadProfile();
|
||||
toast("CV extracted. Review the changes before applying them.", "info");
|
||||
toast(`Queued CV upload (run ${res.data.extractionRunId}).`, "info");
|
||||
} catch (e: any) {
|
||||
toast(String(e?.response?.data || e?.message || t("profileCvUploadFailed")), "error");
|
||||
} finally {
|
||||
@@ -508,7 +531,7 @@ export default function CareerProfilePage() {
|
||||
<Box sx={{ display: "flex", justifyContent: "space-between", gap: 1, flexWrap: "wrap", alignItems: "center", mb: 0.75 }}>
|
||||
<Typography variant="overline">{run.trigger}</Typography>
|
||||
<Box sx={{ display: "flex", gap: 0.75, flexWrap: "wrap" }}>
|
||||
<Chip size="small" label={run.status} color={run.status === "applied" ? "success" : run.status === "failed" ? "error" : "default"} variant={run.status === "applied" ? "filled" : "outlined"} />
|
||||
<Chip size="small" label={cvRunStatus(run)} color={run.status === "applied" ? "success" : run.operation?.status === "failed" || run.status === "failed" ? "error" : "default"} variant={run.status === "applied" ? "filled" : "outlined"} />
|
||||
{run.id === structuredCv.metadata.appliedExtractionRunId ? <Chip size="small" color="primary" label={t("profileCvCurrentRun")} /> : null}
|
||||
</Box>
|
||||
</Box>
|
||||
@@ -521,6 +544,28 @@ export default function CareerProfilePage() {
|
||||
{run.errorMessage}
|
||||
</Typography>
|
||||
) : null}
|
||||
{run.operation?.canCancel ? (
|
||||
<Button size="small" color="inherit" sx={{ mt: 0.75 }} onClick={async () => {
|
||||
try {
|
||||
await api.post(`/operations/${run.operation!.id}/cancel`);
|
||||
await loadProfile();
|
||||
toast("CV processing cancellation requested.", "info");
|
||||
} catch (error) {
|
||||
toast(getApiErrorMessage(error, "Could not cancel CV processing."), "error");
|
||||
}
|
||||
}}>Cancel processing</Button>
|
||||
) : null}
|
||||
{run.operation?.canRetry ? (
|
||||
<Button size="small" color="inherit" sx={{ mt: 0.75 }} onClick={async () => {
|
||||
try {
|
||||
await api.post(`/operations/${run.operation!.id}/retry`);
|
||||
await loadProfile();
|
||||
toast("CV processing queued again.", "info");
|
||||
} catch (error) {
|
||||
toast(getApiErrorMessage(error, "Could not retry CV processing."), "error");
|
||||
}
|
||||
}}>Retry processing</Button>
|
||||
) : null}
|
||||
{run.status === "pending_review" ? (
|
||||
<Box sx={{ mt: 1.25 }}>
|
||||
{runDiffs[run.id] ? (
|
||||
|
||||
Reference in New Issue
Block a user