Merge Request — Wave 0 quick wins + Tier-1/Tier-2 features #1
@@ -48,6 +48,7 @@ tmp/
|
||||
|
||||
# Runtime data that must never be committed (DataProtection keys, exports, CV artifacts)
|
||||
keys/
|
||||
backups/
|
||||
JobTrackerApi/exports/
|
||||
JobTrackerApi/CvArtifacts/
|
||||
JobTrackerApi/CvExports/
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using JobTrackerApi.Services;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Xunit;
|
||||
|
||||
namespace JobTrackerApi.Tests;
|
||||
|
||||
public sealed class DatabaseBackupRunnerTests : IDisposable
|
||||
{
|
||||
private readonly string _root;
|
||||
|
||||
public DatabaseBackupRunnerTests()
|
||||
{
|
||||
_root = Path.Combine(Path.GetTempPath(), $"jt-backup-tests-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(_root);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
SqliteConnection.ClearAllPools();
|
||||
try { Directory.Delete(_root, recursive: true); } catch (IOException) { }
|
||||
}
|
||||
|
||||
private string CreateSourceDb(out string connectionString)
|
||||
{
|
||||
var dbPath = Path.Combine(_root, "source.db");
|
||||
connectionString = $"Data Source={dbPath}";
|
||||
using var connection = new SqliteConnection(connectionString);
|
||||
connection.Open();
|
||||
using var command = connection.CreateCommand();
|
||||
command.CommandText = "CREATE TABLE Sample (Id INTEGER PRIMARY KEY, Name TEXT); INSERT INTO Sample (Name) VALUES ('alpha'), ('beta');";
|
||||
command.ExecuteNonQuery();
|
||||
return dbPath;
|
||||
}
|
||||
|
||||
private SqliteDatabaseBackupRunner CreateRunner(string connectionString, int retainCount = 14)
|
||||
=> new(connectionString, Path.Combine(_root, "backups"), retainCount, NullLogger<SqliteDatabaseBackupRunner>.Instance);
|
||||
|
||||
[Fact]
|
||||
public async Task RunOnce_creates_a_restorable_backup_file()
|
||||
{
|
||||
CreateSourceDb(out var connectionString);
|
||||
var runner = CreateRunner(connectionString);
|
||||
|
||||
var backupPath = await runner.RunOnceAsync(CancellationToken.None);
|
||||
|
||||
Assert.NotNull(backupPath);
|
||||
Assert.True(File.Exists(backupPath));
|
||||
|
||||
await using var verify = new SqliteConnection($"Data Source={backupPath}");
|
||||
await verify.OpenAsync();
|
||||
await using var count = verify.CreateCommand();
|
||||
count.CommandText = "SELECT COUNT(*) FROM Sample";
|
||||
Assert.Equal(2L, (long)(await count.ExecuteScalarAsync())!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RunOnce_prunes_backups_beyond_retention()
|
||||
{
|
||||
CreateSourceDb(out var connectionString);
|
||||
var runner = CreateRunner(connectionString, retainCount: 2);
|
||||
var backupsRoot = runner.BackupsRoot;
|
||||
Directory.CreateDirectory(backupsRoot);
|
||||
|
||||
for (var i = 0; i < 3; i++)
|
||||
{
|
||||
var stale = Path.Combine(backupsRoot, $"{SqliteDatabaseBackupRunner.BackupFilePrefix}stale{i}.db");
|
||||
File.WriteAllText(stale, "stale");
|
||||
File.SetLastWriteTimeUtc(stale, DateTime.UtcNow.AddDays(-10 - i));
|
||||
}
|
||||
|
||||
await runner.RunOnceAsync(CancellationToken.None);
|
||||
|
||||
var remaining = Directory.GetFiles(backupsRoot, $"{SqliteDatabaseBackupRunner.BackupFilePrefix}*.db");
|
||||
Assert.Equal(2, remaining.Length);
|
||||
Assert.Contains(remaining, f => Path.GetFileName(f).Contains("stale0"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Latest_backup_timestamp_reflects_newest_file()
|
||||
{
|
||||
CreateSourceDb(out var connectionString);
|
||||
var runner = CreateRunner(connectionString);
|
||||
|
||||
Assert.Null(runner.GetLatestBackupUtc());
|
||||
|
||||
Directory.CreateDirectory(runner.BackupsRoot);
|
||||
var file = Path.Combine(runner.BackupsRoot, $"{SqliteDatabaseBackupRunner.BackupFilePrefix}x.db");
|
||||
File.WriteAllText(file, "x");
|
||||
var stamp = DateTime.UtcNow.AddHours(-3);
|
||||
File.SetLastWriteTimeUtc(file, stamp);
|
||||
|
||||
var latest = runner.GetLatestBackupUtc();
|
||||
Assert.NotNull(latest);
|
||||
Assert.True(Math.Abs((latest!.Value - stamp).TotalSeconds) < 2);
|
||||
}
|
||||
}
|
||||
@@ -128,6 +128,8 @@ Directory.CreateDirectory(dataProtectionKeysPath);
|
||||
builder.Services.AddDataProtection()
|
||||
.PersistKeysToFileSystem(new DirectoryInfo(dataProtectionKeysPath))
|
||||
.SetApplicationName("JobTracker");
|
||||
builder.Services.AddSingleton<IDatabaseBackupRunner, SqliteDatabaseBackupRunner>();
|
||||
builder.Services.AddHostedService<DatabaseBackupHostedService>();
|
||||
builder.Services.AddHostedService<RulesHostedService>();
|
||||
builder.Services.AddHostedService<FollowUpReminderHostedService>();
|
||||
builder.Services.AddHostedService<DailyExportHostedService>();
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
namespace JobTrackerApi.Services
|
||||
{
|
||||
public sealed class DatabaseBackupHostedService : BackgroundService
|
||||
{
|
||||
private readonly IDatabaseBackupRunner _runner;
|
||||
private readonly ILogger<DatabaseBackupHostedService> _logger;
|
||||
private readonly IConfiguration _cfg;
|
||||
private readonly IStartupReadiness _startupReadiness;
|
||||
|
||||
public DatabaseBackupHostedService(
|
||||
IDatabaseBackupRunner runner,
|
||||
ILogger<DatabaseBackupHostedService> logger,
|
||||
IConfiguration cfg,
|
||||
IStartupReadiness startupReadiness)
|
||||
{
|
||||
_runner = runner;
|
||||
_logger = logger;
|
||||
_cfg = cfg;
|
||||
_startupReadiness = startupReadiness;
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await _startupReadiness.WaitUntilReadyAsync(stoppingToken);
|
||||
|
||||
if (!_cfg.GetValue("Backups:Enabled", true))
|
||||
{
|
||||
_logger.LogInformation("Automated database backups disabled (Backups:Enabled=false).");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_runner.IsSupported)
|
||||
{
|
||||
_logger.LogWarning("Automated database backups are unavailable for the configured provider. Configure external backups for MySQL/MariaDB.");
|
||||
return;
|
||||
}
|
||||
|
||||
var hour = _cfg.GetValue("Backups:HourLocal", 3);
|
||||
if (hour < 0 || hour > 23) hour = 3;
|
||||
|
||||
// Catch-up: guarantee at least one recent backup exists even if the
|
||||
// process never stays up long enough to reach the scheduled hour.
|
||||
var latest = _runner.GetLatestBackupUtc();
|
||||
if (latest is null || latest < DateTime.UtcNow.AddHours(-24))
|
||||
{
|
||||
await TryBackupAsync(stoppingToken);
|
||||
}
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
var now = DateTime.Now;
|
||||
var next = new DateTime(now.Year, now.Month, now.Day, hour, 0, 0);
|
||||
if (next <= now) next = next.AddDays(1);
|
||||
|
||||
_logger.LogInformation("Next database backup scheduled at {Next}.", next);
|
||||
try
|
||||
{
|
||||
await Task.Delay(next - now, stoppingToken);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await TryBackupAsync(stoppingToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task TryBackupAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _runner.RunOnceAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Database backup failed.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
using Microsoft.Data.Sqlite;
|
||||
|
||||
namespace JobTrackerApi.Services
|
||||
{
|
||||
public interface IDatabaseBackupRunner
|
||||
{
|
||||
string BackupsRoot { get; }
|
||||
bool IsSupported { get; }
|
||||
|
||||
/// <summary>Creates one backup file and prunes old ones. Returns the backup path, or null when unsupported.</summary>
|
||||
Task<string?> RunOnceAsync(CancellationToken ct);
|
||||
|
||||
DateTime? GetLatestBackupUtc();
|
||||
}
|
||||
|
||||
public sealed class SqliteDatabaseBackupRunner : IDatabaseBackupRunner
|
||||
{
|
||||
public const string BackupFilePrefix = "jobtracker_backup_";
|
||||
|
||||
private readonly ILogger<SqliteDatabaseBackupRunner> _logger;
|
||||
private readonly string _connectionString;
|
||||
private readonly int _retainCount;
|
||||
|
||||
public string BackupsRoot { get; }
|
||||
public bool IsSupported { get; }
|
||||
|
||||
public SqliteDatabaseBackupRunner(IConfiguration cfg, AppPaths paths, ILogger<SqliteDatabaseBackupRunner> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
|
||||
var provider = (cfg["Database:Provider"] ?? "sqlite").Trim().ToLowerInvariant();
|
||||
var cs = cfg.GetConnectionString("JobTracker");
|
||||
if (string.IsNullOrWhiteSpace(cs))
|
||||
{
|
||||
cs = $"Data Source={paths.GetDbPath()}";
|
||||
provider = "sqlite";
|
||||
}
|
||||
|
||||
_connectionString = cs;
|
||||
IsSupported = provider == "sqlite";
|
||||
BackupsRoot = Path.Combine(paths.DataRoot, "backups");
|
||||
_retainCount = Math.Clamp(cfg.GetValue("Backups:RetainCount", 14), 1, 365);
|
||||
}
|
||||
|
||||
// Test-friendly constructor.
|
||||
public SqliteDatabaseBackupRunner(string connectionString, string backupsRoot, int retainCount, ILogger<SqliteDatabaseBackupRunner> logger)
|
||||
{
|
||||
_logger = logger;
|
||||
_connectionString = connectionString;
|
||||
IsSupported = true;
|
||||
BackupsRoot = backupsRoot;
|
||||
_retainCount = Math.Clamp(retainCount, 1, 365);
|
||||
}
|
||||
|
||||
public async Task<string?> RunOnceAsync(CancellationToken ct)
|
||||
{
|
||||
if (!IsSupported)
|
||||
{
|
||||
_logger.LogWarning("Automated backups only support the SQLite provider. Configure external backups for MySQL/MariaDB.");
|
||||
return null;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(BackupsRoot);
|
||||
|
||||
var target = Path.Combine(BackupsRoot, $"{BackupFilePrefix}{DateTime.UtcNow:yyyyMMdd_HHmmss}.db");
|
||||
if (File.Exists(target)) File.Delete(target);
|
||||
|
||||
await using (var connection = new SqliteConnection(_connectionString))
|
||||
{
|
||||
await connection.OpenAsync(ct);
|
||||
await using var command = connection.CreateCommand();
|
||||
// VACUUM INTO produces a consistent, compacted snapshot without blocking writers (WAL).
|
||||
command.CommandText = $"VACUUM INTO '{target.Replace("'", "''")}'";
|
||||
await command.ExecuteNonQueryAsync(ct);
|
||||
}
|
||||
|
||||
_logger.LogInformation("Database backup written: {File}.", target);
|
||||
PruneOldBackups();
|
||||
return target;
|
||||
}
|
||||
|
||||
public DateTime? GetLatestBackupUtc()
|
||||
{
|
||||
if (!Directory.Exists(BackupsRoot)) return null;
|
||||
var latest = ListBackups().FirstOrDefault();
|
||||
return latest?.LastWriteTimeUtc;
|
||||
}
|
||||
|
||||
private void PruneOldBackups()
|
||||
{
|
||||
foreach (var stale in ListBackups().Skip(_retainCount))
|
||||
{
|
||||
try
|
||||
{
|
||||
stale.Delete();
|
||||
_logger.LogInformation("Pruned old database backup: {File}.", stale.Name);
|
||||
}
|
||||
catch (IOException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Could not prune old database backup {File}.", stale.Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private IOrderedEnumerable<FileInfo> ListBackups()
|
||||
=> new DirectoryInfo(BackupsRoot)
|
||||
.EnumerateFiles($"{BackupFilePrefix}*.db")
|
||||
.OrderByDescending(f => f.LastWriteTimeUtc);
|
||||
}
|
||||
}
|
||||
@@ -133,6 +133,10 @@ Common keys:
|
||||
- `Exports:DailyEnabled`: enable/disable daily export background job
|
||||
- `Exports:DailyFolder`: export destination (relative to `Data:Root` if not absolute)
|
||||
- `Exports:DailyHourLocal`: local hour (0–23) when the daily export runs
|
||||
- `Backups:Enabled`: enable/disable the automated SQLite backup job (default `true`)
|
||||
- `Backups:HourLocal`: local hour (0–23) when the daily database backup runs (default `3`)
|
||||
- `Backups:RetainCount`: how many backup files to keep in `<Data:Root>/backups` (default `14`)
|
||||
- Backups use SQLite `VACUUM INTO` (consistent snapshot, safe with WAL). A catch-up backup runs at startup when none exists from the last 24 h. For MySQL/MariaDB configure external backups instead (see `deploy/MARIADB.md`).
|
||||
- `Auth:GoogleClientId`: if set, enables JWT bearer validation for Google ID tokens
|
||||
- `Auth:JwtKey`: secret used to sign local JWTs for username/password login (set via env var `Auth__JwtKey`)
|
||||
- `Auth:JwtIssuer`: JWT issuer (default `JobTrackerApi`)
|
||||
|
||||
Reference in New Issue
Block a user