From dbf28b97ce0382e115a1ee0c7b269cc5a36a9ed7 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Sat, 15 Aug 2026 20:18:02 +0200 Subject: [PATCH] test(workers): prove clock and restart safety --- .../BackgroundWorkerTenantTests.cs | 109 +++++++++++++++--- .../Services/DailyExportHostedService.cs | 12 +- .../Services/FollowUpReminderHostedService.cs | 9 +- .../Services/JobEnrichmentHostedService.cs | 7 +- JobTrackerApi/Services/RulesHostedService.cs | 11 +- docs/verification/bg-001-tenant-workers.md | 8 +- docs/work-programmes/master-work-plan.md | 4 +- 7 files changed, 124 insertions(+), 36 deletions(-) diff --git a/JobTrackerApi.Tests/BackgroundWorkerTenantTests.cs b/JobTrackerApi.Tests/BackgroundWorkerTenantTests.cs index 664e7b8..0577d41 100644 --- a/JobTrackerApi.Tests/BackgroundWorkerTenantTests.cs +++ b/JobTrackerApi.Tests/BackgroundWorkerTenantTests.cs @@ -18,6 +18,8 @@ namespace JobTrackerApi.Tests; public sealed class BackgroundWorkerTenantTests { + private static readonly DateTimeOffset FixedNow = new(2026, 8, 15, 12, 0, 0, TimeSpan.Zero); + [Fact] public async Task Runner_enters_each_owner_filter_and_isolates_owner_failures() { @@ -64,7 +66,8 @@ public sealed class BackgroundWorkerTenantTests fixture.Runner, fixture.Configuration, NullLogger.Instance, - Mock.Of()); + Mock.Of(), + TimeProvider.System); Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default)); Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default)); @@ -87,10 +90,10 @@ public sealed class BackgroundWorkerTenantTests environment.SetupGet(x => x.ContentRootPath).Returns(root); try { - var rules = new RulesHostedService(runner, configuration, NullLogger.Instance, readiness); - var reminders = new FollowUpReminderHostedService(runner, configuration, NullLogger.Instance, readiness, ExternalOrigin.Parse(null, false)); - var exports = new DailyExportHostedService(runner, NullLogger.Instance, configuration, new AppPaths(configuration, environment.Object), readiness); - var enrichment = new JobEnrichmentHostedService(runner, configuration, NullLogger.Instance, readiness); + var rules = new RulesHostedService(runner, configuration, NullLogger.Instance, readiness, TimeProvider.System); + var reminders = new FollowUpReminderHostedService(runner, configuration, NullLogger.Instance, readiness, ExternalOrigin.Parse(null, false), TimeProvider.System); + var exports = new DailyExportHostedService(runner, NullLogger.Instance, configuration, new AppPaths(configuration, environment.Object), readiness, TimeProvider.System); + var enrichment = new JobEnrichmentHostedService(runner, configuration, NullLogger.Instance, readiness, TimeProvider.System); Assert.Equal(BackgroundWorkerRunResult.Disabled, await rules.RunOnceAsync(default)); Assert.Equal(BackgroundWorkerRunResult.Disabled, await reminders.RunOnceAsync(default)); @@ -114,6 +117,7 @@ public sealed class BackgroundWorkerTenantTests ["Data:Root"] = root, }); await fixture.SeedJobsAsync(); + var clock = new MutableTimeProvider(FixedNow); await using (var seedScope = fixture.Provider.CreateAsyncScope()) { var db = seedScope.ServiceProvider.GetRequiredService(); @@ -135,9 +139,18 @@ public sealed class BackgroundWorkerTenantTests NullLogger.Instance, fixture.Configuration, new AppPaths(fixture.Configuration, environment.Object), - Mock.Of()); + Mock.Of(), + clock); Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default)); + var restartedWorker = new DailyExportHostedService( + fixture.Runner, + NullLogger.Instance, + fixture.Configuration, + new AppPaths(fixture.Configuration, environment.Object), + Mock.Of(), + clock); + Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await restartedWorker.RunOnceAsync(default)); var files = Directory.GetFiles(Path.Combine(root, "exports"), "*.json", SearchOption.AllDirectories); Assert.Equal(2, files.Length); Assert.DoesNotContain(files, path => path.Contains("user-1", StringComparison.Ordinal) || path.Contains("user-2", StringComparison.Ordinal)); @@ -148,6 +161,7 @@ public sealed class BackgroundWorkerTenantTests { using var document = JsonDocument.Parse(System.IO.File.ReadAllText(path)); var rootElement = document.RootElement; + Assert.Equal(FixedNow.DateTime, rootElement.GetProperty("CreatedAt").GetDateTime()); var owner = rootElement.GetProperty("OwnerUserId").GetString(); owners.Add(owner); var attempt = Assert.Single(rootElement.GetProperty("EmailSendAttempts").EnumerateArray()); @@ -206,7 +220,8 @@ public sealed class BackgroundWorkerTenantTests fixture.Runner, fixture.Configuration, NullLogger.Instance, - Mock.Of()); + Mock.Of(), + TimeProvider.System); Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default)); @@ -229,7 +244,8 @@ public sealed class BackgroundWorkerTenantTests fixture.Runner, fixture.Configuration, NullLogger.Instance, - Mock.Of()); + Mock.Of(), + TimeProvider.System); Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default)); summarizer.Verify(x => x.SummarizeAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); @@ -248,20 +264,76 @@ public sealed class BackgroundWorkerTenantTests }, services => services.AddSingleton(email.Object)); await fixture.SeedJobsAsync(includeUsers: true); + var clock = new MutableTimeProvider(FixedNow); var worker = new FollowUpReminderHostedService( fixture.Runner, fixture.Configuration, NullLogger.Instance, Mock.Of(), - ExternalOrigin.FromConfiguration(fixture.Configuration)); + ExternalOrigin.FromConfiguration(fixture.Configuration), + clock); Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await worker.RunOnceAsync(default)); + var restartedWorker = new FollowUpReminderHostedService( + fixture.Runner, + fixture.Configuration, + NullLogger.Instance, + Mock.Of(), + ExternalOrigin.FromConfiguration(fixture.Configuration), + clock); + Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await restartedWorker.RunOnceAsync(default)); email.Verify(x => x.SendAsync("one@example.test", It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); email.Verify(x => x.SendAsync("two@example.test", It.IsAny(), It.IsAny(), It.IsAny()), Times.Once); await using var scope = fixture.Provider.CreateAsyncScope(); var jobs = await scope.ServiceProvider.GetRequiredService().JobApplications.IgnoreQueryFilters().AsNoTracking().ToListAsync(); - Assert.All(jobs, job => Assert.NotNull(job.LastReminderEmailSentAt)); + Assert.All(jobs, job => Assert.Equal(FixedNow.DateTime, job.LastReminderEmailSentAt)); + } + + [Fact] + public async Task Rules_worker_uses_injected_clock_across_threshold_and_restart() + { + var clock = new MutableTimeProvider(FixedNow.AddMinutes(-1)); + await using var fixture = await Fixture.CreateAsync(new Dictionary { ["Workers:RulesEnabled"] = "true" }); + await fixture.SeedJobsAsync(appliedAt: FixedNow.DateTime.AddDays(-5)); + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + var db = scope.ServiceProvider.GetRequiredService(); + db.UserRuleSettings.AddRange( + new UserRuleSettings { OwnerUserId = "user-1", AppliedFollowUpDays = 1, AppliedGhostDays = 5 }, + new UserRuleSettings { OwnerUserId = "user-2", AppliedFollowUpDays = 1, AppliedGhostDays = 5 }); + await db.SaveChangesAsync(); + } + + var beforeBoundary = new RulesHostedService( + fixture.Runner, + fixture.Configuration, + NullLogger.Instance, + Mock.Of(), + clock); + Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await beforeBoundary.RunOnceAsync(default)); + + await using (var scope = fixture.Provider.CreateAsyncScope()) + { + var statuses = await scope.ServiceProvider.GetRequiredService() + .JobApplications.IgnoreQueryFilters().Select(job => job.Status).ToListAsync(); + Assert.All(statuses, status => Assert.Equal("Applied", status)); + } + + clock.SetUtcNow(FixedNow); + var afterRestart = new RulesHostedService( + fixture.Runner, + fixture.Configuration, + NullLogger.Instance, + Mock.Of(), + clock); + Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await afterRestart.RunOnceAsync(default)); + Assert.Equal(new BackgroundWorkerRunResult(true, 2, 2, 0), await afterRestart.RunOnceAsync(default)); + + await using var verificationScope = fixture.Provider.CreateAsyncScope(); + var finalStatuses = await verificationScope.ServiceProvider.GetRequiredService() + .JobApplications.IgnoreQueryFilters().Select(job => job.Status).ToListAsync(); + Assert.All(finalStatuses, status => Assert.Equal("Ghosted", status)); } private sealed class Fixture : IAsyncDisposable @@ -299,7 +371,7 @@ public sealed class BackgroundWorkerTenantTests return new Fixture(connection, provider, configuration); } - public async Task SeedJobsAsync(bool includeUsers = false) + public async Task SeedJobsAsync(bool includeUsers = false, DateTime? appliedAt = null) { await using var scope = Provider.CreateAsyncScope(); var db = scope.ServiceProvider.GetRequiredService(); @@ -317,8 +389,8 @@ public sealed class BackgroundWorkerTenantTests db.Companies.AddRange(companies); await db.SaveChangesAsync(); db.JobApplications.AddRange( - new JobApplication { OwnerUserId = "user-1", CompanyId = companies[0].Id, JobTitle = "One", Status = "Applied", DateApplied = DateTime.Now.AddDays(-30), Description = "description-user-1" }, - new JobApplication { OwnerUserId = "user-2", CompanyId = companies[1].Id, JobTitle = "Two", Status = "Applied", DateApplied = DateTime.Now.AddDays(-30), Description = "description-user-2" }); + new JobApplication { OwnerUserId = "user-1", CompanyId = companies[0].Id, JobTitle = "One", Status = "Applied", DateApplied = appliedAt ?? DateTime.Now.AddDays(-30), Description = "description-user-1" }, + new JobApplication { OwnerUserId = "user-2", CompanyId = companies[1].Id, JobTitle = "Two", Status = "Applied", DateApplied = appliedAt ?? DateTime.Now.AddDays(-30), Description = "description-user-2" }); await db.SaveChangesAsync(); } @@ -328,4 +400,15 @@ public sealed class BackgroundWorkerTenantTests await _connection.DisposeAsync(); } } + + private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _utcNow = utcNow; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public override TimeZoneInfo LocalTimeZone => TimeZoneInfo.Utc; + + public void SetUtcNow(DateTimeOffset value) => _utcNow = value; + } } diff --git a/JobTrackerApi/Services/DailyExportHostedService.cs b/JobTrackerApi/Services/DailyExportHostedService.cs index 5fc6c7e..c2696ac 100644 --- a/JobTrackerApi/Services/DailyExportHostedService.cs +++ b/JobTrackerApi/Services/DailyExportHostedService.cs @@ -10,7 +10,8 @@ public sealed class DailyExportHostedService( ILogger logger, IConfiguration configuration, AppPaths paths, - IStartupReadiness startupReadiness) : BackgroundService + IStartupReadiness startupReadiness, + TimeProvider timeProvider) : BackgroundService { protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -25,11 +26,11 @@ public sealed class DailyExportHostedService( if (hour is < 0 or > 23) hour = 2; while (!stoppingToken.IsCancellationRequested) { - var now = DateTime.Now; + var now = timeProvider.GetLocalNow().DateTime; var next = new DateTime(now.Year, now.Month, now.Day, hour, 0, 0); if (next <= now) next = next.AddDays(1); logger.LogInformation("Next daily export scheduled at {Next}.", next); - await Task.Delay(next - now, stoppingToken); + await Task.Delay(next - now, timeProvider, stoppingToken); await RunOnceAsync(stoppingToken); } } @@ -48,12 +49,13 @@ public sealed class DailyExportHostedService( { var db = services.GetRequiredService(); var owner = db.CurrentUserId ?? throw new InvalidOperationException("Daily export requires an explicit owner scope."); + var now = timeProvider.GetLocalNow().DateTime; var jobs = await db.JobApplications.AsNoTracking().OrderByDescending(job => job.DateApplied).ToListAsync(cancellationToken); var jobIds = jobs.Select(job => job.Id).ToList(); var export = new { Version = "dailyexport.v2", - CreatedAt = DateTime.Now, + CreatedAt = now, OwnerUserId = owner, Companies = await db.Companies.AsNoTracking().OrderBy(company => company.Name).ToListAsync(cancellationToken), JobApplications = jobs, @@ -96,7 +98,7 @@ public sealed class DailyExportHostedService( var folder = paths.GetOwnerDailyExportsRoot(configuration["Exports:DailyFolder"], owner); Directory.CreateDirectory(folder); - var finalPath = Path.Combine(folder, $"daily_export_{DateTime.Now:yyyyMMdd}.json"); + var finalPath = Path.Combine(folder, $"daily_export_{now:yyyyMMdd}.json"); var temporaryPath = finalPath + $".{Guid.NewGuid():N}.tmp"; try { diff --git a/JobTrackerApi/Services/FollowUpReminderHostedService.cs b/JobTrackerApi/Services/FollowUpReminderHostedService.cs index 7febf18..4e2d11f 100644 --- a/JobTrackerApi/Services/FollowUpReminderHostedService.cs +++ b/JobTrackerApi/Services/FollowUpReminderHostedService.cs @@ -10,7 +10,8 @@ public sealed class FollowUpReminderHostedService( IConfiguration configuration, ILogger logger, IStartupReadiness startupReadiness, - ExternalOrigin externalOrigin) : BackgroundService + ExternalOrigin externalOrigin, + TimeProvider timeProvider) : BackgroundService { protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -21,11 +22,11 @@ public sealed class FollowUpReminderHostedService( return; } - await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); + await Task.Delay(TimeSpan.FromSeconds(10), timeProvider, stoppingToken); while (!stoppingToken.IsCancellationRequested) { await RunOnceAsync(stoppingToken); - await Task.Delay(TimeSpan.FromHours(6), stoppingToken); + await Task.Delay(TimeSpan.FromHours(6), timeProvider, stoppingToken); } } @@ -45,7 +46,7 @@ public sealed class FollowUpReminderHostedService( var users = services.GetRequiredService>(); var email = services.GetRequiredService(); var settings = await RulesEngine.GetSettings(db, cancellationToken); - var now = DateTime.Now; + var now = timeProvider.GetLocalNow().DateTime; var lookAheadDays = Math.Clamp(configuration.GetValue("Email:FollowUpReminders:UpcomingDays", 2), 1, 14); var upcomingTo = now.AddDays(lookAheadDays); var lastMessages = await db.Correspondences.AsNoTracking() diff --git a/JobTrackerApi/Services/JobEnrichmentHostedService.cs b/JobTrackerApi/Services/JobEnrichmentHostedService.cs index 4d693e0..37e98c9 100644 --- a/JobTrackerApi/Services/JobEnrichmentHostedService.cs +++ b/JobTrackerApi/Services/JobEnrichmentHostedService.cs @@ -11,7 +11,8 @@ public sealed class JobEnrichmentHostedService( BackgroundTenantRunner tenants, IConfiguration configuration, ILogger logger, - IStartupReadiness startupReadiness) : BackgroundService + IStartupReadiness startupReadiness, + TimeProvider timeProvider) : BackgroundService { protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -22,11 +23,11 @@ public sealed class JobEnrichmentHostedService( return; } - await Task.Delay(TimeSpan.FromSeconds(10), stoppingToken); + await Task.Delay(TimeSpan.FromSeconds(10), timeProvider, stoppingToken); while (!stoppingToken.IsCancellationRequested) { await RunOnceAsync(stoppingToken); - await Task.Delay(TimeSpan.FromMinutes(10), stoppingToken); + await Task.Delay(TimeSpan.FromMinutes(10), timeProvider, stoppingToken); } } diff --git a/JobTrackerApi/Services/RulesHostedService.cs b/JobTrackerApi/Services/RulesHostedService.cs index 875a85f..10b2919 100644 --- a/JobTrackerApi/Services/RulesHostedService.cs +++ b/JobTrackerApi/Services/RulesHostedService.cs @@ -8,7 +8,8 @@ public sealed class RulesHostedService( BackgroundTenantRunner tenants, IConfiguration configuration, ILogger logger, - IStartupReadiness startupReadiness) : BackgroundService + IStartupReadiness startupReadiness, + TimeProvider timeProvider) : BackgroundService { protected override async Task ExecuteAsync(CancellationToken stoppingToken) { @@ -19,11 +20,11 @@ public sealed class RulesHostedService( return; } - await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken); + await Task.Delay(TimeSpan.FromSeconds(2), timeProvider, stoppingToken); while (!stoppingToken.IsCancellationRequested) { await RunOnceAsync(stoppingToken); - await Task.Delay(TimeSpan.FromMinutes(30), stoppingToken); + await Task.Delay(TimeSpan.FromMinutes(30), timeProvider, stoppingToken); } } @@ -35,11 +36,11 @@ public sealed class RulesHostedService( return tenants.RunForJobOwnersAsync("rules", ProcessOwnerAsync, cancellationToken); } - private static async Task ProcessOwnerAsync(IServiceProvider services, CancellationToken cancellationToken) + private async Task ProcessOwnerAsync(IServiceProvider services, CancellationToken cancellationToken) { var db = services.GetRequiredService(); var settings = await RulesEngine.GetSettings(db, cancellationToken); - var now = DateTime.Now; + var now = timeProvider.GetLocalNow().DateTime; var lastMessages = await db.Correspondences .GroupBy(message => message.JobApplicationId) .Select(group => new { JobApplicationId = group.Key, Last = group.Max(x => x.Date) }) diff --git a/docs/verification/bg-001-tenant-workers.md b/docs/verification/bg-001-tenant-workers.md index e9607b8..64c03f5 100644 --- a/docs/verification/bg-001-tenant-workers.md +++ b/docs/verification/bg-001-tenant-workers.md @@ -1,6 +1,6 @@ # BG-001 tenant-safe worker foundation verification -Updated: 2026-08-02 +Updated: 2026-08-15 Status: `IMPLEMENTED — NOT VERIFIED`. The owner-scoping foundation and default-off activation contract pass local tests. Browser, production canary, durable notification/idempotency and multi-replica gates remain. @@ -16,12 +16,12 @@ The four repaired workers are deny-by-default through new switches. Old email/ex | Check | Result | |---|---| -| `dotnet test ... --filter "FullyQualifiedName~BackgroundWorkerTenantTests|FullyQualifiedName~CurrentUserIdLiveEvaluationTests|FullyQualifiedName~RulesEngineTests"` | PASS — 9/9 after final trust-boundary test | +| `dotnet test ... --filter "FullyQualifiedName~BackgroundWorkerTenantTests"` | PASS — 9/9, including fixed-clock boundary and restart cases | | `dotnet test JobTrackerApi.Tests/JobTrackerApi.Tests.csproj --no-restore` | PASS — 532/532 after the final trust-boundary test | | `docker compose config --quiet` | PASS; only unset optional/local environment warnings | | `git diff --check` | PASS — no whitespace errors; repository line-ending notices only | -Real SQLite tests prove two-owner/no-HTTP filtering, owner-failure isolation, per-owner rules with an idempotent second pass, atomic per-owner daily exports with hashed filenames, fake-AI enrichment for both owners, fake-email reminders for both confirmed owners, default-off behavior for all four workers and HTTP-context override refusal. +Real SQLite tests prove two-owner/no-HTTP filtering, owner-failure isolation, per-owner rules at an exact fixed-clock threshold with an idempotent post-restart pass, atomic per-owner daily exports that overwrite the same date file after restart, fake-AI enrichment for both owners, fake-email reminders that do not resend after restart, default-off behavior for all four workers and HTTP-context override refusal. All four worker loops and business-date decisions now use the injected `TimeProvider` rather than hidden system clocks. ## Runtime evidence @@ -32,5 +32,5 @@ An isolated app using disposable data under `docs/audits/evidence/bg-001-runtime - Reminder delivery is not exactly-once across an email-success/database-failure boundary. Keep it off until OPS-001 supplies a persistent notification/outbox operation. - AI enrichment must remain off until POL-001/POL-002 and durable AI operations are enforced server-side. - Rules and export remain off pending notification/audit and retention/operator rollout respectively. -- No lease, heartbeat, distributed scheduler, restart/clock-boundary suite, browser surface or production canary was added here. +- No lease, heartbeat, distributed scheduler, browser surface or production canary was added here. Local restart and clock-boundary coverage is complete; multi-replica coordination remains an activation concern rather than default-off foundation work. - Rollback is setting all four worker switches false, then reverting the runner/service changes. Do not delete export files or undo user-visible mutations without a separate reviewed procedure. No schema migration was introduced. diff --git a/docs/work-programmes/master-work-plan.md b/docs/work-programmes/master-work-plan.md index 01eca3f..7ec50b9 100644 --- a/docs/work-programmes/master-work-plan.md +++ b/docs/work-programmes/master-work-plan.md @@ -324,9 +324,9 @@ This queue records the highest-value work that can proceed without production cr - **Required production verification:** one-worker canary and owner-safe metrics. - **Status:** `IMPLEMENTED — NOT VERIFIED`. - **Blocker:** activation blocked until policy/notification prerequisites; foundation code is not blocked. -- **Evidence:** audit JT-005 service inspection; `docs/verification/bg-001-tenant-workers.md`; real-SQLite two-owner worker suite with fake email/AI; full backend 532/532; Compose validation; isolated default-off startup/health/no-export check. +- **Evidence:** audit JT-005 service inspection; `docs/verification/bg-001-tenant-workers.md`; real-SQLite two-owner worker suite with fake email/AI, exact fixed-clock threshold, fresh-instance restart idempotency and reminder/export deduplication; Compose validation; isolated default-off startup/health/no-export check. - **Commit:** none. -- **Remaining work:** OPS-001B persistent notification/idempotency before reminders/rules; POL-001/002 and durable AI queue before enrichment; restart/clock tests; browser result surfaces; monitored single-worker production canary. Keep all four switches false. +- **Remaining work:** activation policy prerequisites, browser result surfaces and a monitored single-worker production canary. Local restart/clock coverage is complete. Keep all four switches false. ### OPS-001A — Durable operation record and lease state machine