test(workers): prove clock and restart safety

JobTrackerApi.Tests/BackgroundWorkerTenantTests.cs

@@ -18,6 +18,8 @@ namespace JobTrackerApi.Tests;
 
 public sealed class BackgroundWorkerTenantTests
 {
+    private static readonly DateTimeOffset FixedNow = new(2026, 8, 15, 12, 0, 0, TimeSpan.Zero);
+
     [Fact]
     public async Task Runner_enters_each_owner_filter_and_isolates_owner_failures()
     {
@@ -64,7 +66,8 @@ public sealed class BackgroundWorkerTenantTests
             fixture.Runner,
             fixture.Configuration,
             NullLogger<RulesHostedService>.Instance,
-            Mock.Of<IStartupReadiness>());
+            Mock.Of<IStartupReadiness>(),
+            TimeProvider.System);
 
         Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default));
         Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default));
@@ -87,10 +90,10 @@ public sealed class BackgroundWorkerTenantTests
         environment.SetupGet(x => x.ContentRootPath).Returns(root);
         try
         {
-            var rules = new RulesHostedService(runner, configuration, NullLogger<RulesHostedService>.Instance, readiness);
+            var rules = new RulesHostedService(runner, configuration, NullLogger<RulesHostedService>.Instance, readiness, TimeProvider.System);
-            var reminders = new FollowUpReminderHostedService(runner, configuration, NullLogger<FollowUpReminderHostedService>.Instance, readiness, ExternalOrigin.Parse(null, false));
+            var reminders = new FollowUpReminderHostedService(runner, configuration, NullLogger<FollowUpReminderHostedService>.Instance, readiness, ExternalOrigin.Parse(null, false), TimeProvider.System);
-            var exports = new DailyExportHostedService(runner, NullLogger<DailyExportHostedService>.Instance, configuration, new AppPaths(configuration, environment.Object), readiness);
+            var exports = new DailyExportHostedService(runner, NullLogger<DailyExportHostedService>.Instance, configuration, new AppPaths(configuration, environment.Object), readiness, TimeProvider.System);
-            var enrichment = new JobEnrichmentHostedService(runner, configuration, NullLogger<JobEnrichmentHostedService>.Instance, readiness);
+            var enrichment = new JobEnrichmentHostedService(runner, configuration, NullLogger<JobEnrichmentHostedService>.Instance, readiness, TimeProvider.System);
 
             Assert.Equal(BackgroundWorkerRunResult.Disabled, await rules.RunOnceAsync(default));
             Assert.Equal(BackgroundWorkerRunResult.Disabled, await reminders.RunOnceAsync(default));
@@ -114,6 +117,7 @@ public sealed class BackgroundWorkerTenantTests
             ["Data:Root"] = root,
         });
         await fixture.SeedJobsAsync();
+        var clock = new MutableTimeProvider(FixedNow);
         await using (var seedScope = fixture.Provider.CreateAsyncScope())
         {
             var db = seedScope.ServiceProvider.GetRequiredService<JobTrackerContext>();
@@ -135,9 +139,18 @@ public sealed class BackgroundWorkerTenantTests
                 NullLogger<DailyExportHostedService>.Instance,
                 fixture.Configuration,
                 new AppPaths(fixture.Configuration, environment.Object),
-                Mock.Of<IStartupReadiness>());
+                Mock.Of<IStartupReadiness>(),
+                clock);
 
             Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default));
+            var restartedWorker = new DailyExportHostedService(
+                fixture.Runner,
+                NullLogger<DailyExportHostedService>.Instance,
+                fixture.Configuration,
+                new AppPaths(fixture.Configuration, environment.Object),
+                Mock.Of<IStartupReadiness>(),
+                clock);
+            Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await restartedWorker.RunOnceAsync(default));
             var files = Directory.GetFiles(Path.Combine(root, "exports"), "*.json", SearchOption.AllDirectories);
             Assert.Equal(2, files.Length);
             Assert.DoesNotContain(files, path => path.Contains("user-1", StringComparison.Ordinal) || path.Contains("user-2", StringComparison.Ordinal));
@@ -148,6 +161,7 @@ public sealed class BackgroundWorkerTenantTests
             {
                 using var document = JsonDocument.Parse(System.IO.File.ReadAllText(path));
                 var rootElement = document.RootElement;
+                Assert.Equal(FixedNow.DateTime, rootElement.GetProperty("CreatedAt").GetDateTime());
                 var owner = rootElement.GetProperty("OwnerUserId").GetString();
                 owners.Add(owner);
                 var attempt = Assert.Single(rootElement.GetProperty("EmailSendAttempts").EnumerateArray());
@@ -206,7 +220,8 @@ public sealed class BackgroundWorkerTenantTests
             fixture.Runner,
             fixture.Configuration,
             NullLogger<JobEnrichmentHostedService>.Instance,
-            Mock.Of<IStartupReadiness>());
+            Mock.Of<IStartupReadiness>(),
+            TimeProvider.System);
 
         Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default));
 
@@ -229,7 +244,8 @@ public sealed class BackgroundWorkerTenantTests
             fixture.Runner,
             fixture.Configuration,
             NullLogger<JobEnrichmentHostedService>.Instance,
-            Mock.Of<IStartupReadiness>());
+            Mock.Of<IStartupReadiness>(),
+            TimeProvider.System);
 
         Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default));
         summarizer.Verify(x => x.SummarizeAsync(It.IsAny<string>(), It.IsAny<int>(), It.IsAny<int>()), Times.Never);
@@ -248,20 +264,76 @@ public sealed class BackgroundWorkerTenantTests
             },
             services => services.AddSingleton(email.Object));
         await fixture.SeedJobsAsync(includeUsers: true);
+        var clock = new MutableTimeProvider(FixedNow);
         var worker = new FollowUpReminderHostedService(
             fixture.Runner,
             fixture.Configuration,
             NullLogger<FollowUpReminderHostedService>.Instance,
             Mock.Of<IStartupReadiness>(),
-            ExternalOrigin.FromConfiguration(fixture.Configuration));
+            ExternalOrigin.FromConfiguration(fixture.Configuration),
+            clock);
 
         Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default));
+        var restartedWorker = new FollowUpReminderHostedService(
+            fixture.Runner,
+            fixture.Configuration,
+            NullLogger<FollowUpReminderHostedService>.Instance,
+            Mock.Of<IStartupReadiness>(),
+            ExternalOrigin.FromConfiguration(fixture.Configuration),
+            clock);
+        Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await restartedWorker.RunOnceAsync(default));
         email.Verify(x => x.SendAsync("one@example.test", It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once);
         email.Verify(x => x.SendAsync("two@example.test", It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()), Times.Once);
 
         await using var scope = fixture.Provider.CreateAsyncScope();
         var jobs = await scope.ServiceProvider.GetRequiredService<JobTrackerContext>().JobApplications.IgnoreQueryFilters().AsNoTracking().ToListAsync();
-        Assert.All(jobs, job => Assert.NotNull(job.LastReminderEmailSentAt));
+        Assert.All(jobs, job => Assert.Equal(FixedNow.DateTime, job.LastReminderEmailSentAt));
+    }
+
+    [Fact]
+    public async Task Rules_worker_uses_injected_clock_across_threshold_and_restart()
+    {
+        var clock = new MutableTimeProvider(FixedNow.AddMinutes(-1));
+        await using var fixture = await Fixture.CreateAsync(new Dictionary<string, string?> { ["Workers:RulesEnabled"] = "true" });
+        await fixture.SeedJobsAsync(appliedAt: FixedNow.DateTime.AddDays(-5));
+        await using (var scope = fixture.Provider.CreateAsyncScope())
+        {
+            var db = scope.ServiceProvider.GetRequiredService<JobTrackerContext>();
+            db.UserRuleSettings.AddRange(
+                new UserRuleSettings { OwnerUserId = "user-1", AppliedFollowUpDays = 1, AppliedGhostDays = 5 },
+                new UserRuleSettings { OwnerUserId = "user-2", AppliedFollowUpDays = 1, AppliedGhostDays = 5 });
+            await db.SaveChangesAsync();
+        }
+
+        var beforeBoundary = new RulesHostedService(
+            fixture.Runner,
+            fixture.Configuration,
+            NullLogger<RulesHostedService>.Instance,
+            Mock.Of<IStartupReadiness>(),
+            clock);
+        Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await beforeBoundary.RunOnceAsync(default));
+
+        await using (var scope = fixture.Provider.CreateAsyncScope())
+        {
+            var statuses = await scope.ServiceProvider.GetRequiredService<JobTrackerContext>()
+                .JobApplications.IgnoreQueryFilters().Select(job => job.Status).ToListAsync();
+            Assert.All(statuses, status => Assert.Equal("Applied", status));
+        }
+
+        clock.SetUtcNow(FixedNow);
+        var afterRestart = new RulesHostedService(
+            fixture.Runner,
+            fixture.Configuration,
+            NullLogger<RulesHostedService>.Instance,
+            Mock.Of<IStartupReadiness>(),
+            clock);
+        Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await afterRestart.RunOnceAsync(default));
+        Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await afterRestart.RunOnceAsync(default));
+
+        await using var verificationScope = fixture.Provider.CreateAsyncScope();
+        var finalStatuses = await verificationScope.ServiceProvider.GetRequiredService<JobTrackerContext>()
+            .JobApplications.IgnoreQueryFilters().Select(job => job.Status).ToListAsync();
+        Assert.All(finalStatuses, status => Assert.Equal("Ghosted", status));
     }
 
     private sealed class Fixture : IAsyncDisposable
@@ -299,7 +371,7 @@ public sealed class BackgroundWorkerTenantTests
             return new Fixture(connection, provider, configuration);
         }
 
-        public async Task SeedJobsAsync(bool includeUsers = false)
+        public async Task SeedJobsAsync(bool includeUsers = false, DateTime? appliedAt = null)
         {
             await using var scope = Provider.CreateAsyncScope();
             var db = scope.ServiceProvider.GetRequiredService<JobTrackerContext>();
@@ -317,8 +389,8 @@ public sealed class BackgroundWorkerTenantTests
             db.Companies.AddRange(companies);
             await db.SaveChangesAsync();
             db.JobApplications.AddRange(
-                new JobApplication { OwnerUserId = "user-1", CompanyId = companies[0].Id, JobTitle = "One", Status = "Applied", DateApplied = DateTime.Now.AddDays(-30), Description = "description-user-1" },
+                new JobApplication { OwnerUserId = "user-1", CompanyId = companies[0].Id, JobTitle = "One", Status = "Applied", DateApplied = appliedAt ?? DateTime.Now.AddDays(-30), Description = "description-user-1" },
-                new JobApplication { OwnerUserId = "user-2", CompanyId = companies[1].Id, JobTitle = "Two", Status = "Applied", DateApplied = DateTime.Now.AddDays(-30), Description = "description-user-2" });
+                new JobApplication { OwnerUserId = "user-2", CompanyId = companies[1].Id, JobTitle = "Two", Status = "Applied", DateApplied = appliedAt ?? DateTime.Now.AddDays(-30), Description = "description-user-2" });
             await db.SaveChangesAsync();
         }
 
@@ -328,4 +400,15 @@ public sealed class BackgroundWorkerTenantTests
             await _connection.DisposeAsync();
         }
     }
+
+    private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider
+    {
+        private DateTimeOffset _utcNow = utcNow;
+
+        public override DateTimeOffset GetUtcNow() => _utcNow;
+
+        public override TimeZoneInfo LocalTimeZone => TimeZoneInfo.Utc;
+
+        public void SetUtcNow(DateTimeOffset value) => _utcNow = value;
+    }
 }

JobTrackerApi/Services/DailyExportHostedService.cs

@@ -10,7 +10,8 @@ public sealed class DailyExportHostedService(
     ILogger<DailyExportHostedService> logger,
     IConfiguration configuration,
     AppPaths paths,
-    IStartupReadiness startupReadiness) : BackgroundService
+    IStartupReadiness startupReadiness,
+    TimeProvider timeProvider) : BackgroundService
 {
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
@@ -25,11 +26,11 @@ public sealed class DailyExportHostedService(
         if (hour is < 0 or > 23) hour = 2;
         while (!stoppingToken.IsCancellationRequested)
         {
-            var now = DateTime.Now;
+            var now = timeProvider.GetLocalNow().DateTime;
             var next = new DateTime(now.Year, now.Month, now.Day, hour, 0, 0);
             if (next <= now) next = next.AddDays(1);
             logger.LogInformation("Next daily export scheduled at {Next}.", next);
-            await Task.Delay(next - now, stoppingToken);
+            await Task.Delay(next - now, timeProvider, stoppingToken);
             await RunOnceAsync(stoppingToken);
         }
     }
@@ -48,12 +49,13 @@ public sealed class DailyExportHostedService(
     {
         var db = services.GetRequiredService<JobTrackerContext>();
         var owner = db.CurrentUserId ?? throw new InvalidOperationException("Daily export requires an explicit owner scope.");
+        var now = timeProvider.GetLocalNow().DateTime;
         var jobs = await db.JobApplications.AsNoTracking().OrderByDescending(job => job.DateApplied).ToListAsync(cancellationToken);
         var jobIds = jobs.Select(job => job.Id).ToList();
         var export = new
         {
             Version = "dailyexport.v2",
-            CreatedAt = DateTime.Now,
+            CreatedAt = now,
             OwnerUserId = owner,
             Companies = await db.Companies.AsNoTracking().OrderBy(company => company.Name).ToListAsync(cancellationToken),
             JobApplications = jobs,
@@ -96,7 +98,7 @@ public sealed class DailyExportHostedService(
 
         var folder = paths.GetOwnerDailyExportsRoot(configuration["Exports:DailyFolder"], owner);
         Directory.CreateDirectory(folder);
-        var finalPath = Path.Combine(folder, $"daily_export_{DateTime.Now:yyyyMMdd}.json");
+        var finalPath = Path.Combine(folder, $"daily_export_{now:yyyyMMdd}.json");
         var temporaryPath = finalPath + $".{Guid.NewGuid():N}.tmp";
         try
         {

JobTrackerApi/Services/FollowUpReminderHostedService.cs

@@ -10,7 +10,8 @@ public sealed class FollowUpReminderHostedService(
     IConfiguration configuration,
     ILogger<FollowUpReminderHostedService> logger,
     IStartupReadiness startupReadiness,
-    ExternalOrigin externalOrigin) : BackgroundService
+    ExternalOrigin externalOrigin,
+    TimeProvider timeProvider) : BackgroundService
 {
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
@@ -21,11 +22,11 @@ public sealed class FollowUpReminderHostedService(
             return;
         }
 
-        await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
+        await Task.Delay(TimeSpan.FromSeconds(10), timeProvider, stoppingToken);
         while (!stoppingToken.IsCancellationRequested)
         {
             await RunOnceAsync(stoppingToken);
-            await Task.Delay(TimeSpan.FromHours(6), stoppingToken);
+            await Task.Delay(TimeSpan.FromHours(6), timeProvider, stoppingToken);
         }
     }
 
@@ -45,7 +46,7 @@ public sealed class FollowUpReminderHostedService(
         var users = services.GetRequiredService<UserManager<ApplicationUser>>();
         var email = services.GetRequiredService<IAppEmailSender>();
         var settings = await RulesEngine.GetSettings(db, cancellationToken);
-        var now = DateTime.Now;
+        var now = timeProvider.GetLocalNow().DateTime;
         var lookAheadDays = Math.Clamp(configuration.GetValue("Email:FollowUpReminders:UpcomingDays", 2), 1, 14);
         var upcomingTo = now.AddDays(lookAheadDays);
         var lastMessages = await db.Correspondences.AsNoTracking()

JobTrackerApi/Services/JobEnrichmentHostedService.cs

@@ -11,7 +11,8 @@ public sealed class JobEnrichmentHostedService(
     BackgroundTenantRunner tenants,
     IConfiguration configuration,
     ILogger<JobEnrichmentHostedService> logger,
-    IStartupReadiness startupReadiness) : BackgroundService
+    IStartupReadiness startupReadiness,
+    TimeProvider timeProvider) : BackgroundService
 {
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
@@ -22,11 +23,11 @@ public sealed class JobEnrichmentHostedService(
             return;
         }
 
-        await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken);
+        await Task.Delay(TimeSpan.FromSeconds(10), timeProvider, stoppingToken);
         while (!stoppingToken.IsCancellationRequested)
         {
             await RunOnceAsync(stoppingToken);
-            await Task.Delay(TimeSpan.FromMinutes(10), stoppingToken);
+            await Task.Delay(TimeSpan.FromMinutes(10), timeProvider, stoppingToken);
         }
     }
 

JobTrackerApi/Services/RulesHostedService.cs

@@ -8,7 +8,8 @@ public sealed class RulesHostedService(
     BackgroundTenantRunner tenants,
     IConfiguration configuration,
     ILogger<RulesHostedService> logger,
-    IStartupReadiness startupReadiness) : BackgroundService
+    IStartupReadiness startupReadiness,
+    TimeProvider timeProvider) : BackgroundService
 {
     protected override async Task ExecuteAsync(CancellationToken stoppingToken)
     {
@@ -19,11 +20,11 @@ public sealed class RulesHostedService(
             return;
         }
 
-        await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken);
+        await Task.Delay(TimeSpan.FromSeconds(2), timeProvider, stoppingToken);
         while (!stoppingToken.IsCancellationRequested)
         {
             await RunOnceAsync(stoppingToken);
-            await Task.Delay(TimeSpan.FromMinutes(30), stoppingToken);
+            await Task.Delay(TimeSpan.FromMinutes(30), timeProvider, stoppingToken);
         }
     }
 
@@ -35,11 +36,11 @@ public sealed class RulesHostedService(
         return tenants.RunForJobOwnersAsync("rules", ProcessOwnerAsync, cancellationToken);
     }
 
-    private static async Task ProcessOwnerAsync(IServiceProvider services, CancellationToken cancellationToken)
+    private async Task ProcessOwnerAsync(IServiceProvider services, CancellationToken cancellationToken)
     {
         var db = services.GetRequiredService<JobTrackerContext>();
         var settings = await RulesEngine.GetSettings(db, cancellationToken);
-        var now = DateTime.Now;
+        var now = timeProvider.GetLocalNow().DateTime;
         var lastMessages = await db.Correspondences
             .GroupBy(message => message.JobApplicationId)
             .Select(group => new { JobApplicationId = group.Key, Last = group.Max(x => x.Date) })

docs/verification/bg-001-tenant-workers.md

@@ -1,6 +1,6 @@
 # BG-001 tenant-safe worker foundation verification
 
-Updated: 2026-08-02
+Updated: 2026-08-15
 
 Status: `IMPLEMENTED — NOT VERIFIED`. The owner-scoping foundation and default-off activation contract pass local tests. Browser, production canary, durable notification/idempotency and multi-replica gates remain.
 
@@ -16,12 +16,12 @@ The four repaired workers are deny-by-default through new switches. Old email/ex
 
 | Check | Result |
 |---|---|
-| `dotnet test ... --filter "FullyQualifiedName~BackgroundWorkerTenantTests|FullyQualifiedName~CurrentUserIdLiveEvaluationTests|FullyQualifiedName~RulesEngineTests"` | PASS — 9/9 after final trust-boundary test |
+| `dotnet test ... --filter "FullyQualifiedName~BackgroundWorkerTenantTests"` | PASS — 9/9, including fixed-clock boundary and restart cases |
 | `dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --no-restore` | PASS — 532/532 after the final trust-boundary test |
 | `docker compose config --quiet` | PASS; only unset optional/local environment warnings |
 | `git diff --check` | PASS — no whitespace errors; repository line-ending notices only |
 
-Real SQLite tests prove two-owner/no-HTTP filtering, owner-failure isolation, per-owner rules with an idempotent second pass, atomic per-owner daily exports with hashed filenames, fake-AI enrichment for both owners, fake-email reminders for both confirmed owners, default-off behavior for all four workers and HTTP-context override refusal.
+Real SQLite tests prove two-owner/no-HTTP filtering, owner-failure isolation, per-owner rules at an exact fixed-clock threshold with an idempotent post-restart pass, atomic per-owner daily exports that overwrite the same date file after restart, fake-AI enrichment for both owners, fake-email reminders that do not resend after restart, default-off behavior for all four workers and HTTP-context override refusal. All four worker loops and business-date decisions now use the injected `TimeProvider` rather than hidden system clocks.
 
 ## Runtime evidence
 
@@ -32,5 +32,5 @@ An isolated app using disposable data under `docs/audits/evidence/bg-001-runtime
 - Reminder delivery is not exactly-once across an email-success/database-failure boundary. Keep it off until OPS-001 supplies a persistent notification/outbox operation.
 - AI enrichment must remain off until POL-001/POL-002 and durable AI operations are enforced server-side.
 - Rules and export remain off pending notification/audit and retention/operator rollout respectively.
-- No lease, heartbeat, distributed scheduler, restart/clock-boundary suite, browser surface or production canary was added here.
+- No lease, heartbeat, distributed scheduler, browser surface or production canary was added here. Local restart and clock-boundary coverage is complete; multi-replica coordination remains an activation concern rather than default-off foundation work.
 - Rollback is setting all four worker switches false, then reverting the runner/service changes. Do not delete export files or undo user-visible mutations without a separate reviewed procedure. No schema migration was introduced.

docs/work-programmes/master-work-plan.md

@@ -324,9 +324,9 @@ This queue records the highest-value work that can proceed without production cr
 - **Required production verification:** one-worker canary and owner-safe metrics.
 - **Status:** `IMPLEMENTED — NOT VERIFIED`.
 - **Blocker:** activation blocked until policy/notification prerequisites; foundation code is not blocked.
-- **Evidence:** audit JT-005 service inspection; `docs/verification/bg-001-tenant-workers.md`; real-SQLite two-owner worker suite with fake email/AI; full backend 532/532; Compose validation; isolated default-off startup/health/no-export check.
+- **Evidence:** audit JT-005 service inspection; `docs/verification/bg-001-tenant-workers.md`; real-SQLite two-owner worker suite with fake email/AI, exact fixed-clock threshold, fresh-instance restart idempotency and reminder/export deduplication; Compose validation; isolated default-off startup/health/no-export check.
 - **Commit:** none.
-- **Remaining work:** OPS-001B persistent notification/idempotency before reminders/rules; POL-001/002 and durable AI queue before enrichment; restart/clock tests; browser result surfaces; monitored single-worker production canary. Keep all four switches false.
+- **Remaining work:** activation policy prerequisites, browser result surfaces and a monitored single-worker production canary. Local restart/clock coverage is complete. Keep all four switches false.
 
 ### OPS-001A — Durable operation record and lease state machine