From ee5ef7e12aa9d41db84ecc5a86c45296e06df926 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Mon, 10 Aug 2026 00:16:56 +0200 Subject: [PATCH] fix(email): reconcile abandoned sends Age stale pre-delivery attempts to failed and in-delivery attempts to uncertain without provider I/O. Notify each owner with content-free guidance and keep recovery idempotent across replicas and restarts. --- .../EmailSendAttemptStoreTests.cs | 68 +++++++++++++++++-- JobTrackerApi/Program.cs | 1 + .../EmailSendAttemptRecoveryHostedService.cs | 44 ++++++++++++ .../Services/EmailSendAttemptStore.cs | 54 +++++++++++++++ 4 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 JobTrackerApi/Services/EmailSendAttemptRecoveryHostedService.cs diff --git a/JobTrackerApi.Tests/EmailSendAttemptStoreTests.cs b/JobTrackerApi.Tests/EmailSendAttemptStoreTests.cs index 656a3f7..b75a155 100644 --- a/JobTrackerApi.Tests/EmailSendAttemptStoreTests.cs +++ b/JobTrackerApi.Tests/EmailSendAttemptStoreTests.cs @@ -58,12 +58,49 @@ public sealed class EmailSendAttemptStoreTests property.Name.Contains("Recipient", StringComparison.OrdinalIgnoreCase)); } + [Fact] + public async Task Restart_recovery_is_tenant_visible_idempotent_and_never_requeues_delivery() + { + await using var fixture = await Fixture.CreateAsync(); + var ownerOne = fixture.Store("user-1"); + var ownerTwo = fixture.Store("user-2"); + var oldSending = await ownerOne.CreateAsync(new(fixture.JobId, "gmail", Guid.NewGuid().ToString(), Hash('e')), default); + var oldPending = await ownerTwo.CreateAsync(new(fixture.OtherJobId, "microsoft", Guid.NewGuid().ToString(), Hash('f')), default); + Assert.Equal(1, await ownerOne.BeginAsync(oldSending.Attempt.Id, default)); + + fixture.Time.Advance(EmailSendAttemptStore.AbandonedAge + TimeSpan.FromSeconds(1)); + var freshPending = await ownerOne.CreateAsync(new(fixture.JobId, "gmail", Guid.NewGuid().ToString(), Hash('1')), default); + var freshSending = await ownerTwo.CreateAsync(new(fixture.OtherJobId, "microsoft", Guid.NewGuid().ToString(), Hash('2')), default); + Assert.Equal(1, await ownerTwo.BeginAsync(freshSending.Attempt.Id, default)); + + var recovered = await fixture.Store(null).ReconcileAbandonedAsync(default); + var repeated = await fixture.Store(null).ReconcileAbandonedAsync(default); + + Assert.Equal(new EmailSendAttemptRecoveryResult(1, 1), recovered); + Assert.Equal(new EmailSendAttemptRecoveryResult(0, 0), repeated); + Assert.Equal(EmailSendStatuses.Uncertain, (await ownerOne.GetAsync(oldSending.Attempt.Id, default))!.Status); + Assert.Equal(EmailSendStatuses.Failed, (await ownerTwo.GetAsync(oldPending.Attempt.Id, default))!.Status); + Assert.Equal(EmailSendStatuses.Pending, (await ownerOne.GetAsync(freshPending.Attempt.Id, default))!.Status); + Assert.Equal(EmailSendStatuses.Sending, (await ownerTwo.GetAsync(freshSending.Attempt.Id, default))!.Status); + + var userOneNotifications = await fixture.Notifications("user-1").ListAsync(10, default); + var userTwoNotifications = await fixture.Notifications("user-2").ListAsync(10, default); + Assert.Single(userOneNotifications); + Assert.Single(userTwoNotifications); + Assert.Equal("email.send.uncertain", userOneNotifications[0].Kind); + Assert.Equal("email.send.stopped", userTwoNotifications[0].Kind); + Assert.DoesNotContain("gmail", userOneNotifications[0].Message, StringComparison.OrdinalIgnoreCase); + Assert.DoesNotContain("microsoft", userTwoNotifications[0].Message, StringComparison.OrdinalIgnoreCase); + } + private static string Hash(char value) => new(value, 64); - private sealed class Fixture(SqliteConnection connection, DbContextOptions options, int jobId) : IAsyncDisposable + private sealed class Fixture(SqliteConnection connection, DbContextOptions options, int jobId, int otherJobId) : IAsyncDisposable { private readonly List contexts = new(); + public ManualTimeProvider Time { get; } = new(new DateTimeOffset(2026, 8, 10, 12, 0, 0, TimeSpan.Zero)); public int JobId { get; } = jobId; + public int OtherJobId { get; } = otherJobId; public static async Task CreateAsync() { @@ -78,17 +115,31 @@ public sealed class EmailSendAttemptStoreTests var job = new JobApplication { JobTitle = "Backend", CompanyId = company.Id, OwnerUserId = "user-1" }; db.JobApplications.Add(job); await db.SaveChangesAsync(); - return new Fixture(connection, options, job.Id); + await using var otherDb = CreateDb(options, "user-2"); + var otherCompany = new Company { Name = "Other", OwnerUserId = "user-2" }; + otherDb.Companies.Add(otherCompany); + await otherDb.SaveChangesAsync(); + var otherJob = new JobApplication { JobTitle = "Frontend", CompanyId = otherCompany.Id, OwnerUserId = "user-2" }; + otherDb.JobApplications.Add(otherJob); + await otherDb.SaveChangesAsync(); + return new Fixture(connection, options, job.Id, otherJob.Id); } - public EmailSendAttemptStore Store(string userId) + public EmailSendAttemptStore Store(string? userId) { var db = CreateDb(options, userId); contexts.Add(db); - return new EmailSendAttemptStore(db, TimeProvider.System); + return new EmailSendAttemptStore(db, Time); } - private static JobTrackerContext CreateDb(DbContextOptions options, string userId) + public UserNotificationStore Notifications(string userId) + { + var db = CreateDb(options, userId); + contexts.Add(db); + return new UserNotificationStore(db, Time); + } + + private static JobTrackerContext CreateDb(DbContextOptions options, string? userId) { var currentUser = new Mock(); currentUser.SetupGet(service => service.UserId).Returns(userId); @@ -101,4 +152,11 @@ public sealed class EmailSendAttemptStoreTests await connection.DisposeAsync(); } } + + private sealed class ManualTimeProvider(DateTimeOffset now) : TimeProvider + { + public DateTimeOffset UtcNow { get; private set; } = now; + public override DateTimeOffset GetUtcNow() => UtcNow; + public void Advance(TimeSpan duration) => UtcNow += duration; + } } diff --git a/JobTrackerApi/Program.cs b/JobTrackerApi/Program.cs index fb07049..e9c8109 100644 --- a/JobTrackerApi/Program.cs +++ b/JobTrackerApi/Program.cs @@ -168,6 +168,7 @@ builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); +builder.Services.AddHostedService(); builder.Services.AddHttpClient("jobimport") .ConfigurePrimaryHttpMessageHandler(() => new HttpClientHandler diff --git a/JobTrackerApi/Services/EmailSendAttemptRecoveryHostedService.cs b/JobTrackerApi/Services/EmailSendAttemptRecoveryHostedService.cs new file mode 100644 index 0000000..4b41851 --- /dev/null +++ b/JobTrackerApi/Services/EmailSendAttemptRecoveryHostedService.cs @@ -0,0 +1,44 @@ +namespace JobTrackerApi.Services; + +public sealed class EmailSendAttemptRecoveryHostedService( + IServiceScopeFactory scopes, + IStartupReadiness startupReadiness, + ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await startupReadiness.WaitUntilReadyAsync(stoppingToken); + while (!stoppingToken.IsCancellationRequested) + { + try + { + await using var scope = scopes.CreateAsyncScope(); + var result = await scope.ServiceProvider.GetRequiredService() + .ReconcileAbandonedAsync(stoppingToken); + if (result.FailedPending > 0 || result.UncertainSending > 0) + { + logger.LogWarning( + "Recovered abandoned email attempts: failedPending={FailedPending}, uncertainSending={UncertainSending}. No provider retry was attempted.", + result.FailedPending, result.UncertainSending); + } + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + logger.LogError(ex, "Email send-attempt recovery failed; provider delivery was not attempted."); + } + + try + { + await Task.Delay(TimeSpan.FromMinutes(5), stoppingToken); + } + catch (OperationCanceledException) + { + break; + } + } + } +} diff --git a/JobTrackerApi/Services/EmailSendAttemptStore.cs b/JobTrackerApi/Services/EmailSendAttemptStore.cs index f68af79..179a59d 100644 --- a/JobTrackerApi/Services/EmailSendAttemptStore.cs +++ b/JobTrackerApi/Services/EmailSendAttemptStore.cs @@ -6,11 +6,13 @@ namespace JobTrackerApi.Services; public sealed record CreateEmailSendAttempt(int JobApplicationId, string Provider, string ClientRequestId, string PayloadHash); public sealed record EmailSendAttemptCreation(EmailSendAttempt Attempt, bool Created); +public sealed record EmailSendAttemptRecoveryResult(int FailedPending, int UncertainSending); public sealed class EmailSendConflictException(string message) : InvalidOperationException(message); public sealed class EmailSendAttemptStore(JobTrackerContext db, TimeProvider timeProvider) { + public static readonly TimeSpan AbandonedAge = TimeSpan.FromMinutes(15); private DateTime UtcNow => timeProvider.GetUtcNow().UtcDateTime; public Task GetAsync(Guid id, CancellationToken cancellationToken) @@ -74,6 +76,58 @@ public sealed class EmailSendAttemptStore(JobTrackerContext db, TimeProvider tim public Task MarkUncertainAsync(Guid id, string failureCategory, CancellationToken cancellationToken) => CompleteAsync(id, EmailSendStatuses.Uncertain, null, failureCategory, cancellationToken); + public async Task ReconcileAbandonedAsync(CancellationToken cancellationToken) + { + var now = UtcNow; + var cutoff = now - AbandonedAge; + var candidates = await db.EmailSendAttempts.IgnoreQueryFilters().AsNoTracking() + .Where(item => + (item.Status == EmailSendStatuses.Pending && item.CreatedAtUtc <= cutoff) || + (item.Status == EmailSendStatuses.Sending && (item.StartedAtUtc ?? item.CreatedAtUtc) <= cutoff)) + .Select(item => new { item.Id, item.OwnerUserId, item.Status }) + .ToListAsync(cancellationToken); + if (candidates.Count == 0) return new EmailSendAttemptRecoveryResult(0, 0); + + await using var transaction = db.Database.IsRelational() + ? await db.Database.BeginTransactionAsync(cancellationToken) + : null; + var failedPending = 0; + var uncertainSending = 0; + foreach (var candidate in candidates) + { + var failed = candidate.Status == EmailSendStatuses.Pending; + var terminalStatus = failed ? EmailSendStatuses.Failed : EmailSendStatuses.Uncertain; + var failureCategory = failed ? "process_stopped_before_delivery" : "process_interrupted"; + var affected = await db.EmailSendAttempts.IgnoreQueryFilters() + .Where(item => item.Id == candidate.Id && item.Status == candidate.Status && + (failed ? item.CreatedAtUtc <= cutoff : (item.StartedAtUtc ?? item.CreatedAtUtc) <= cutoff)) + .ExecuteUpdateAsync(setters => setters + .SetProperty(item => item.Status, terminalStatus) + .SetProperty(item => item.FailureCategory, failureCategory) + .SetProperty(item => item.CompletedAtUtc, now), cancellationToken); + if (affected != 1) continue; + + if (failed) failedPending++; + else uncertainSending++; + db.UserNotifications.Add(new UserNotification + { + Id = Guid.NewGuid(), + OwnerUserId = candidate.OwnerUserId, + Kind = failed ? "email.send.stopped" : "email.send.uncertain", + Title = failed ? "Email send stopped" : "Check email delivery", + Message = failed + ? "An email attempt stopped before provider delivery. Review the draft before trying again." + : "Email delivery could not be confirmed after an interruption. Check the provider Sent folder and do not resend automatically.", + LinkPath = "/correspondence", + CreatedAtUtc = now, + }); + } + + await db.SaveChangesAsync(cancellationToken); + if (transaction is not null) await transaction.CommitAsync(cancellationToken); + return new EmailSendAttemptRecoveryResult(failedPending, uncertainSending); + } + private Task CompleteAsync(Guid id, string status, string? providerMessageId, string? failureCategory, CancellationToken cancellationToken) { EnsureOwnerScope();