fix(email): reconcile abandoned sends

Age stale pre-delivery attempts to failed and in-delivery attempts to uncertain without provider I/O. Notify each owner with content-free guidance and keep recovery idempotent across replicas and restarts.

JobTrackerApi.Tests/EmailSendAttemptStoreTests.cs

@@ -58,12 +58,49 @@ public sealed class EmailSendAttemptStoreTests
             property.Name.Contains("Recipient", StringComparison.OrdinalIgnoreCase));
     }
 
+    [Fact]
+    public async Task Restart_recovery_is_tenant_visible_idempotent_and_never_requeues_delivery()
+    {
+        await using var fixture = await Fixture.CreateAsync();
+        var ownerOne = fixture.Store("user-1");
+        var ownerTwo = fixture.Store("user-2");
+        var oldSending = await ownerOne.CreateAsync(new(fixture.JobId, "gmail", Guid.NewGuid().ToString(), Hash('e')), default);
+        var oldPending = await ownerTwo.CreateAsync(new(fixture.OtherJobId, "microsoft", Guid.NewGuid().ToString(), Hash('f')), default);
+        Assert.Equal(1, await ownerOne.BeginAsync(oldSending.Attempt.Id, default));
+
+        fixture.Time.Advance(EmailSendAttemptStore.AbandonedAge + TimeSpan.FromSeconds(1));
+        var freshPending = await ownerOne.CreateAsync(new(fixture.JobId, "gmail", Guid.NewGuid().ToString(), Hash('1')), default);
+        var freshSending = await ownerTwo.CreateAsync(new(fixture.OtherJobId, "microsoft", Guid.NewGuid().ToString(), Hash('2')), default);
+        Assert.Equal(1, await ownerTwo.BeginAsync(freshSending.Attempt.Id, default));
+
+        var recovered = await fixture.Store(null).ReconcileAbandonedAsync(default);
+        var repeated = await fixture.Store(null).ReconcileAbandonedAsync(default);
+
+        Assert.Equal(new EmailSendAttemptRecoveryResult(1, 1), recovered);
+        Assert.Equal(new EmailSendAttemptRecoveryResult(0, 0), repeated);
+        Assert.Equal(EmailSendStatuses.Uncertain, (await ownerOne.GetAsync(oldSending.Attempt.Id, default))!.Status);
+        Assert.Equal(EmailSendStatuses.Failed, (await ownerTwo.GetAsync(oldPending.Attempt.Id, default))!.Status);
+        Assert.Equal(EmailSendStatuses.Pending, (await ownerOne.GetAsync(freshPending.Attempt.Id, default))!.Status);
+        Assert.Equal(EmailSendStatuses.Sending, (await ownerTwo.GetAsync(freshSending.Attempt.Id, default))!.Status);
+
+        var userOneNotifications = await fixture.Notifications("user-1").ListAsync(10, default);
+        var userTwoNotifications = await fixture.Notifications("user-2").ListAsync(10, default);
+        Assert.Single(userOneNotifications);
+        Assert.Single(userTwoNotifications);
+        Assert.Equal("email.send.uncertain", userOneNotifications[0].Kind);
+        Assert.Equal("email.send.stopped", userTwoNotifications[0].Kind);
+        Assert.DoesNotContain("gmail", userOneNotifications[0].Message, StringComparison.OrdinalIgnoreCase);
+        Assert.DoesNotContain("microsoft", userTwoNotifications[0].Message, StringComparison.OrdinalIgnoreCase);
+    }
+
     private static string Hash(char value) => new(value, 64);
 
-    private sealed class Fixture(SqliteConnection connection, DbContextOptions<JobTrackerContext> options, int jobId) : IAsyncDisposable
+    private sealed class Fixture(SqliteConnection connection, DbContextOptions<JobTrackerContext> options, int jobId, int otherJobId) : IAsyncDisposable
     {
         private readonly List<JobTrackerContext> contexts = new();
+        public ManualTimeProvider Time { get; } = new(new DateTimeOffset(2026, 8, 10, 12, 0, 0, TimeSpan.Zero));
         public int JobId { get; } = jobId;
+        public int OtherJobId { get; } = otherJobId;
 
         public static async Task<Fixture> CreateAsync()
         {
@@ -78,17 +115,31 @@ public sealed class EmailSendAttemptStoreTests
             var job = new JobApplication { JobTitle = "Backend", CompanyId = company.Id, OwnerUserId = "user-1" };
             db.JobApplications.Add(job);
             await db.SaveChangesAsync();
-            return new Fixture(connection, options, job.Id);
+            await using var otherDb = CreateDb(options, "user-2");
+            var otherCompany = new Company { Name = "Other", OwnerUserId = "user-2" };
+            otherDb.Companies.Add(otherCompany);
+            await otherDb.SaveChangesAsync();
+            var otherJob = new JobApplication { JobTitle = "Frontend", CompanyId = otherCompany.Id, OwnerUserId = "user-2" };
+            otherDb.JobApplications.Add(otherJob);
+            await otherDb.SaveChangesAsync();
+            return new Fixture(connection, options, job.Id, otherJob.Id);
         }
 
-        public EmailSendAttemptStore Store(string userId)
+        public EmailSendAttemptStore Store(string? userId)
         {
             var db = CreateDb(options, userId);
             contexts.Add(db);
-            return new EmailSendAttemptStore(db, TimeProvider.System);
+            return new EmailSendAttemptStore(db, Time);
         }
 
-        private static JobTrackerContext CreateDb(DbContextOptions<JobTrackerContext> options, string userId)
+        public UserNotificationStore Notifications(string userId)
+        {
+            var db = CreateDb(options, userId);
+            contexts.Add(db);
+            return new UserNotificationStore(db, Time);
+        }
+
+        private static JobTrackerContext CreateDb(DbContextOptions<JobTrackerContext> options, string? userId)
         {
             var currentUser = new Mock<ICurrentUserService>();
             currentUser.SetupGet(service => service.UserId).Returns(userId);
@@ -101,4 +152,11 @@ public sealed class EmailSendAttemptStoreTests
             await connection.DisposeAsync();
         }
     }
+
+    private sealed class ManualTimeProvider(DateTimeOffset now) : TimeProvider
+    {
+        public DateTimeOffset UtcNow { get; private set; } = now;
+        public override DateTimeOffset GetUtcNow() => UtcNow;
+        public void Advance(TimeSpan duration) => UtcNow += duration;
+    }
 }