diff --git a/.env.example b/.env.example index b39d939..1b0218c 100644 --- a/.env.example +++ b/.env.example @@ -98,6 +98,12 @@ MAINTENANCE_GITEA_BASE_URL=https://git.cesnimda.uk MAINTENANCE_GITEA_OWNER=cesnimda MAINTENANCE_GITEA_REPOSITORY=jobtrackingapp MAINTENANCE_GITEA_TOKEN= +# Complete recovery sets are created before deployment and nightly at 02:00 Europe/Oslo. +BACKUP_TIMEZONE=Europe/Oslo +BACKUP_DAILY_RETENTION_DAYS=30 +BACKUP_MONTHLY_RETENTION_COUNT=12 +# Optional encrypted rclone destination, for example gdrive-crypt:jobtracker. +BACKUP_RCLONE_REMOTE= EMAIL_ENABLED=false EMAIL_SMTP_HOST=smtp.gmail.com EMAIL_SMTP_PORT=587 diff --git a/.gitea/workflows/backup.yml b/.gitea/workflows/backup.yml new file mode 100644 index 0000000..b0c5629 --- /dev/null +++ b/.gitea/workflows/backup.yml @@ -0,0 +1,32 @@ +name: Production backup + +on: + schedule: + # One of these UTC hours is 02:00 in Europe/Oslo; the guard skips the other. + - cron: '0 0,1 * * *' + workflow_dispatch: + +jobs: + backup: + runs-on: ubuntu-latest + steps: + - name: Create verified production recovery set + uses: appleboy/ssh-action@v1.0.3 + with: + host: ${{ secrets.PROD_HOST }} + username: ${{ secrets.PROD_USER }} + key: ${{ secrets.PROD_SSH_KEY }} + command_timeout: 20m + script: | + set -euo pipefail + if [ "${{ github.event_name }}" = "schedule" ] \ + && [ "$(TZ=Europe/Oslo date +%H)" != "02" ]; then + echo "Outside the 02:00 Europe/Oslo backup window." + exit 0 + fi + cd /opt/job-tracker/app + if [ ! -x deploy/deploy.sh ]; then + echo "The deployed backup script is unavailable." + exit 1 + fi + DEPLOY_BACKUP_ONLY=true deploy/deploy.sh diff --git a/.gitea/workflows/maintenance.yml b/.gitea/workflows/maintenance.yml index 8acae32..6f4effd 100644 --- a/.gitea/workflows/maintenance.yml +++ b/.gitea/workflows/maintenance.yml @@ -15,28 +15,48 @@ jobs: if: inputs.job == 'cv-benchmark' runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup .NET - uses: actions/setup-dotnet@v4 + - name: Run private corpus benchmark on production + uses: appleboy/ssh-action@v1.0.3 with: - dotnet-version: 9.0.317 + host: ${{ secrets.PROD_HOST }} + username: ${{ secrets.PROD_USER }} + key: ${{ secrets.PROD_SSH_KEY }} + command_timeout: 30m + script: | + set -euo pipefail + cd /opt/job-tracker/app + corpus_dir=/home/pi/cvs + approved_dir="$corpus_dir/approved-jsons" + output_dir="$(mktemp -d)/latest" + cleanup() { rm -rf "$(dirname "$output_dir")"; } + trap cleanup EXIT - - name: Run CV corpus benchmark - env: - CV_BENCHMARK_OUTPUT_DIR: ${{ github.workspace }}/tmp/cv-benchmarks/latest - CV_BENCHMARK_APPROVED_DIR: ${{ github.workspace }}/tmp/cv-benchmarks/approved-fixtures - run: | - chmod +x scripts/run-cv-benchmark.sh - scripts/run-cv-benchmark.sh - test -s "$CV_BENCHMARK_OUTPUT_DIR/index.json" - test -s "$CV_BENCHMARK_OUTPUT_DIR/report.md" + AI_SERVICE_TOKEN="$(sed -n 's/^AI_SERVICE_TOKEN=//p' /opt/job-tracker/shared/.env | tail -n 1)" + if [ -z "$AI_SERVICE_TOKEN" ]; then + echo "AI_SERVICE_TOKEN is not configured." >&2 + exit 1 + fi + ai_id="$(docker compose -f docker-compose.yml ps -q ai-service)" + ai_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$ai_id")" + if [ -z "$ai_ip" ]; then + echo "The AI extraction service is not running." >&2 + exit 1 + fi - - name: Retain benchmark report and fixture candidates - uses: actions/upload-artifact@v4 - with: - name: cv-benchmark-${{ github.run_number }} - path: tmp/cv-benchmarks - if-no-files-found: error - retention-days: 30 + chmod +x scripts/run-cv-benchmark.sh + CV_BENCHMARK_CORPUS_DIR="$corpus_dir" \ + CV_BENCHMARK_OUTPUT_DIR="$output_dir" \ + CV_BENCHMARK_APPROVED_DIR="$approved_dir" \ + CV_AI_BASE_URL="http://$ai_ip:8001" \ + CV_AI_SERVICE_TOKEN="$AI_SERVICE_TOKEN" \ + scripts/run-cv-benchmark.sh + + backend_id="$(docker compose -f docker-compose.yml ps -q backend)" + if [ -z "$backend_id" ]; then + echo "The backend container is not running." >&2 + exit 1 + fi + docker exec "$backend_id" sh -c 'mkdir -p /data/CvBenchmarks' + docker cp "$output_dir/." "$backend_id:/data/CvBenchmarks/" + docker exec "$backend_id" test -s /data/CvBenchmarks/index.json + docker exec "$backend_id" test -s /data/CvBenchmarks/report.md diff --git a/JobTrackerApi.Tests/CvCorpusHarnessTests.cs b/JobTrackerApi.Tests/CvCorpusHarnessTests.cs index 37d0ce0..009e874 100644 --- a/JobTrackerApi.Tests/CvCorpusHarnessTests.cs +++ b/JobTrackerApi.Tests/CvCorpusHarnessTests.cs @@ -11,6 +11,7 @@ using Microsoft.AspNetCore.Identity; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Caching.Memory; using Moq; using Xunit; @@ -52,24 +53,18 @@ public sealed class CvCorpusHarnessTests userManager.Setup(x => x.GetUserAsync(It.IsAny())).ReturnsAsync(user); userManager.Setup(x => x.UpdateAsync(It.IsAny())).ReturnsAsync(IdentityResult.Success); - var aiService = new Mock(); - aiService.Setup(x => x.SummarizeSectionAsync(It.Is(instruction => instruction.Contains("Extract this CV into structured JSON", StringComparison.Ordinal)), It.IsAny(), 3200, 900)).ReturnsAsync(string.Empty); - aiService.Setup(x => x.SummarizeSectionAsync(It.Is(instruction => instruction.Contains("Reconstruct this CV text extracted from a PDF", StringComparison.Ordinal)), It.IsAny(), 2800, 900)).ReturnsAsync((string _, string text, int _, int __) => text); + var aiService = CreateSummarizerFromEnvironment(); var cvAiNormalizer = CreateCvAiNormalizerFromEnvironment(); await using var db = TestHostFactory.CreateInMemoryDb(); var paths = CreatePaths(outputRoot); - var controller = new ProfileCvController(userManager.Object, aiService.Object, db, paths, null, NoOpCvAiClassifier.Instance, cvAiNormalizer) + var controller = new ProfileCvController(userManager.Object, aiService, db, paths, null, NoOpCvAiClassifier.Instance, cvAiNormalizer) { ControllerContext = new ControllerContext { HttpContext = new DefaultHttpContext() } }; - var extractMethod = typeof(ProfileCvController).GetMethod("ExtractTextAsync", BindingFlags.NonPublic | BindingFlags.Static); - var reconstructMethod = typeof(ProfileCvController).GetMethod("MaybeReconstructStructuredCvAsync", BindingFlags.NonPublic | BindingFlags.Instance); - var buildMethod = typeof(ProfileCvController).GetMethod("BuildStructuredCvAsync", BindingFlags.NonPublic | BindingFlags.Instance); + var extractMethod = typeof(ProfileCvController).GetMethod("ExtractStructuredCvFromFileAsync", BindingFlags.NonPublic | BindingFlags.Instance); Assert.NotNull(extractMethod); - Assert.NotNull(reconstructMethod); - Assert.NotNull(buildMethod); var entries = new List(); foreach (var path in files) @@ -83,16 +78,14 @@ public sealed class CvCorpusHarnessTests }; var extension = Path.GetExtension(path); - var extractTask = (Task)extractMethod!.Invoke(null, new object[] { formFile, extension })!; - var text = await extractTask; + var extractTask = (Task)extractMethod!.Invoke(controller, new object[] { formFile, extension, CancellationToken.None })!; + await extractTask; + var extraction = extractTask.GetType().GetProperty("Result")!.GetValue(extractTask)!; + var resultType = extraction.GetType(); + var text = (string)resultType.GetProperty("RawText")!.GetValue(extraction)!; Assert.False(string.IsNullOrWhiteSpace(text)); - - var reconstructTask = (Task)reconstructMethod!.Invoke(controller, new object[] { text, CancellationToken.None })!; - var normalizedText = await reconstructTask; - Assert.False(string.IsNullOrWhiteSpace(normalizedText)); - - var buildTask = (Task)buildMethod!.Invoke(controller, new object[] { normalizedText, CancellationToken.None })!; - var structured = StructuredCvProfileJson.Normalize(await buildTask); + var structured = StructuredCvProfileJson.Normalize( + (StructuredCvProfile)resultType.GetProperty("StructuredCv")!.GetValue(extraction)!); Assert.NotNull(structured); var slug = Slugify(fileName); @@ -382,9 +375,44 @@ public sealed class CvCorpusHarnessTests { client.BaseAddress = new Uri(baseUrl.Trim()); client.Timeout = TimeSpan.FromSeconds(180); + var token = Environment.GetEnvironmentVariable("CV_AI_SERVICE_TOKEN"); + if (!string.IsNullOrWhiteSpace(token)) client.DefaultRequestHeaders.Add("X-Ai-Service-Token", token); }); var provider = services.BuildServiceProvider(); var factory = provider.GetRequiredService(); return new CvAiNormalizer(factory); } + + private static ISummarizerService CreateSummarizerFromEnvironment() + { + var baseUrl = Environment.GetEnvironmentVariable("CV_AI_BASE_URL"); + if (string.IsNullOrWhiteSpace(baseUrl)) + { + var mock = new Mock(); + mock.Setup(x => x.ExtractTextAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(async (Stream stream, string fileName, string? contentType, CancellationToken cancellationToken) => + { + using var reader = new StreamReader(stream); + var text = await reader.ReadToEndAsync(cancellationToken); + return new AiTextExtractionResult(text, false, contentType ?? "text/plain", null, text.Length, fileName); + }); + mock.Setup(x => x.SummarizeSectionAsync(It.Is(instruction => instruction.Contains("Extract this CV into structured JSON", StringComparison.Ordinal)), It.IsAny(), 3200, 900)).ReturnsAsync(string.Empty); + mock.Setup(x => x.SummarizeSectionAsync(It.Is(instruction => instruction.Contains("Reconstruct this CV text extracted from a PDF", StringComparison.Ordinal)), It.IsAny(), 2800, 900)).ReturnsAsync((string _, string text, int _, int __) => text); + return mock.Object; + } + + var services = new Microsoft.Extensions.DependencyInjection.ServiceCollection(); + services.AddMemoryCache(); + services.AddHttpClient("ai-service", client => + { + client.BaseAddress = new Uri(baseUrl.Trim()); + client.Timeout = TimeSpan.FromMinutes(3); + var token = Environment.GetEnvironmentVariable("CV_AI_SERVICE_TOKEN"); + if (!string.IsNullOrWhiteSpace(token)) client.DefaultRequestHeaders.Add("X-Ai-Service-Token", token); + }); + var provider = services.BuildServiceProvider(); + return new SummarizerService( + provider.GetRequiredService(), + provider.GetRequiredService()); + } } diff --git a/JobTrackerApi.Tests/Fixtures/CvCorpus/synthetic-developer-cv.txt b/JobTrackerApi.Tests/Fixtures/CvCorpus/synthetic-developer-cv.txt new file mode 100644 index 0000000..690aa96 --- /dev/null +++ b/JobTrackerApi.Tests/Fixtures/CvCorpus/synthetic-developer-cv.txt @@ -0,0 +1,24 @@ +Alex Example +Software Developer +alex.example@example.test | +44 20 7946 0000 | London, United Kingdom +https://example.test/alex + +Professional Summary +Software developer building reliable web applications and APIs. + +Technical Skills +C#, .NET, TypeScript, SQL, Docker + +Professional Experience +Software Developer | Example Systems Ltd | London, United Kingdom +January 2022 - Present +- Built accessible web application features using .NET and TypeScript. +- Improved automated test coverage for critical application workflows. + +Education +BSc Computer Science | Example University | London, United Kingdom +September 2018 - June 2021 + +Languages +English - Native +Norwegian Bokmal - Intermediate diff --git a/JobTrackerApi/Services/MaintenanceJobDispatcher.cs b/JobTrackerApi/Services/MaintenanceJobDispatcher.cs index 3da83d1..4af8954 100644 --- a/JobTrackerApi/Services/MaintenanceJobDispatcher.cs +++ b/JobTrackerApi/Services/MaintenanceJobDispatcher.cs @@ -17,7 +17,7 @@ public sealed class MaintenanceJobDispatcher(IConfiguration configuration, IHttp new( CvBenchmarkJobId, "CV extraction benchmark", - "Runs the CV corpus benchmark and retains the report and fixture candidates as a workflow artifact.", + "Runs the private CV corpus benchmark on the production host and publishes its report to Admin/System.", settings.IsConfigured, settings.RunsUrl) ]; diff --git a/deploy/README.md b/deploy/README.md index 65a28a5..b3fc51d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -236,9 +236,13 @@ MYSQL_PWD='' mariadb-dump \ ### Retention -**Nothing is deleted automatically.** Backups accumulate in `BACKUP_DIR` until you remove them. Watch -disk usage and prune deliberately — a suggested policy is to keep every backup for 30 days and one per -month after that, but the script does not enforce it and will not delete your files. +Verified top-level recovery sets are retained for 30 days and the first complete set in each month is +copied into `monthly/`; the newest 12 monthly sets are retained. Configure +`BACKUP_DAILY_RETENTION_DAYS` and `BACKUP_MONTHLY_RETENTION_COUNT` to change those positive integer +limits. Paths outside the fixed backup directory and monthly naming contract are never pruned. + +Set `BACKUP_RCLONE_REMOTE` to an encrypted rclone remote to copy each current recovery set off-host. +See [`docs/operations/google-drive-backups.md`](../docs/operations/google-drive-backups.md). ## Restoring the database diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 2bc35d6..e826786 100644 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -265,7 +265,7 @@ validate_deploy_config() { # exactly when a restore point matters. See deploy/README.md. # --------------------------------------------------------------------------- BACKUP_DIR="${BACKUP_DIR:-/opt/job-tracker/backups}" -BACKUP_STAMP="$(date -u +%Y%m%dT%H%M%SZ)" +BACKUP_STAMP="${DEPLOY_BACKUP_STAMP:-$(date -u +%Y%m%dT%H%M%SZ)}" backup_database() { if [ "${DEPLOY_SKIP_DB_BACKUP:-false}" = "true" ]; then @@ -406,9 +406,9 @@ verify_backup_file_shape() { report_backup() { local target="$1" size + chmod 600 "$target" size="$(du -h "$target" | cut -f1)" echo "Backup verified: ${target} (${size})" - echo "Retention is manual — old backups in ${BACKUP_DIR} are never deleted automatically." return 0 } @@ -513,6 +513,74 @@ backup_recovery_volumes() { "deletion-tombstones-${project_name}" false } +apply_backup_retention() { + local daily_days="${BACKUP_DAILY_RETENTION_DAYS:-30}" + local monthly_count="${BACKUP_MONTHLY_RETENTION_COUNT:-12}" + local timezone="${BACKUP_TIMEZONE:-Europe/Oslo}" + if [[ ! "$daily_days" =~ ^[1-9][0-9]*$ ]] || [[ ! "$monthly_count" =~ ^[1-9][0-9]*$ ]]; then + echo "Backup retention must use positive integer day/count values." + return 1 + fi + + local month monthly_root monthly_dir + month="$(TZ="$timezone" date +%Y-%m)" + monthly_root="$BACKUP_DIR/monthly" + monthly_dir="$monthly_root/$month" + mkdir -p "$monthly_root" + chmod 700 "$monthly_root" + + local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*) + if [ ! -e "${current_files[0]}" ]; then + echo "No verified backup files were found for stamp ${BACKUP_STAMP}." + return 1 + fi + + if [ ! -f "$monthly_dir/.complete" ]; then + mkdir -p "$monthly_dir" + chmod 700 "$monthly_dir" + cp --preserve=mode,timestamps -- "${current_files[@]}" "$monthly_dir/" + printf '%s\n' "$BACKUP_STAMP" > "$monthly_dir/.complete" + chmod 600 "$monthly_dir/.complete" + echo "Monthly recovery set retained: ${monthly_dir}" + fi + + find "$BACKUP_DIR" -maxdepth 1 -type f -name 'jobtracker-*' -mtime "+$daily_days" -delete + + local monthly_dirs=() + mapfile -t monthly_dirs < <(find "$monthly_root" -mindepth 1 -maxdepth 1 -type d -name '20??-??' | sort) + while [ "${#monthly_dirs[@]}" -gt "$monthly_count" ]; do + local oldest="${monthly_dirs[0]}" + case "$oldest" in + "$monthly_root"/20??-??) rm -rf -- "$oldest" ;; + *) echo "Refusing to prune unexpected monthly backup path: $oldest"; return 1 ;; + esac + monthly_dirs=("${monthly_dirs[@]:1}") + done + + echo "Backup retention applied: ${daily_days} days daily, ${monthly_count} monthly sets." +} + +upload_current_backup_set() { + local remote="${BACKUP_RCLONE_REMOTE:-}" + [ -n "$remote" ] || return 0 + if ! command -v rclone >/dev/null 2>&1; then + echo "BACKUP_RCLONE_REMOTE is configured but rclone is not installed." + return 1 + fi + + remote="${remote%/}" + local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*) file + for file in "${current_files[@]}"; do + rclone copyto "$file" "$remote/daily/$(basename "$file")" --immutable + done + + local month="$(TZ="${BACKUP_TIMEZONE:-Europe/Oslo}" date +%Y-%m)" + if [ -d "$BACKUP_DIR/monthly/$month" ]; then + rclone copy "$BACKUP_DIR/monthly/$month" "$remote/monthly/$month" --immutable + fi + echo "Current recovery set copied to configured encrypted off-host remote." +} + resolve_existing_web_proxy_subnet if ! validate_deploy_config; then @@ -527,6 +595,15 @@ if ! backup_recovery_volumes; then exit 1 fi +if ! apply_backup_retention || ! upload_current_backup_set; then + exit 1 +fi + +if [ "${DEPLOY_BACKUP_ONLY:-false}" = "true" ]; then + echo "Backup-only run complete: ${BACKUP_STAMP}" + exit 0 +fi + previous_backend_ref="" previous_frontend_ref="" capture_previous_core_image() { diff --git a/docs/operations/google-drive-backups.md b/docs/operations/google-drive-backups.md new file mode 100644 index 0000000..a943150 --- /dev/null +++ b/docs/operations/google-drive-backups.md @@ -0,0 +1,48 @@ +# Encrypted Google Drive backups + +Yes—JobTracker recovery sets can be copied to Google Drive. Use an `rclone crypt` remote so database, +CV, attachment and key archives are encrypted before they leave the production host. + +## One-time production setup + +1. Install `rclone` from the operating system's trusted package source. +2. Run `rclone config` as the same operating-system user used by deployment. +3. Create a Google Drive remote, for example `gdrive`. +4. Complete Google OAuth in your browser. Do not paste its refresh token into Git, chat or CI. +5. Create a second remote of type `crypt`, for example `gdrive-crypt`, targeting a dedicated folder + such as `gdrive:JobTrackerBackups`. +6. Enable filename and directory-name encryption. Store the crypt password and salt in the protected + production credential store/rclone configuration, not in the repository. +7. Verify the encrypted remote without uploading private application data: + + ```bash + printf 'backup connectivity test\n' >/tmp/jobtracker-backup-test.txt + rclone copy /tmp/jobtracker-backup-test.txt gdrive-crypt:jobtracker-test + rclone ls gdrive-crypt:jobtracker-test + rm -f /tmp/jobtracker-backup-test.txt + rclone purge gdrive-crypt:jobtracker-test + ``` + +8. Add this to `/opt/job-tracker/shared/.env`: + + ```text + BACKUP_RCLONE_REMOTE=gdrive-crypt:jobtracker + ``` + +9. Run the **Production backup** workflow manually. It must create the local verified recovery set + first, copy that set to Drive, and fail visibly if the off-host copy fails. + +## Schedule and retention + +- Scheduled workflow: 00:00 and 01:00 UTC triggers with a Europe/Oslo guard; one backup runs at local + 02:00 even when daylight-saving time changes. +- Recovery-point objective: 24 hours. +- Local retention: 30 days plus 12 monthly recovery sets. +- Google Drive retention: the upload is immutable. Apply a matching Drive/rclone lifecycle only after + a scratch restore has proven the off-host copy. + +## Recovery rule + +Download into a scratch directory and restore into a separate MariaDB/database-volume environment. +Never rehearse by restoring over live production. Preserve the newest deletion-tombstone ledger when +restoring older application data. diff --git a/scripts/run-cv-benchmark.sh b/scripts/run-cv-benchmark.sh index bc725e2..4d38e9a 100755 --- a/scripts/run-cv-benchmark.sh +++ b/scripts/run-cv-benchmark.sh @@ -4,16 +4,32 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" OUTPUT_DIR="${CV_BENCHMARK_OUTPUT_DIR:-$ROOT_DIR/tmp/cv-benchmarks/latest}" APPROVED_DIR="${CV_BENCHMARK_APPROVED_DIR:-$ROOT_DIR/tmp/cv-benchmarks/approved-fixtures}" +CORPUS_DIR="${CV_BENCHMARK_CORPUS_DIR:-/home/pi/cvs}" + +if [[ ! -d "$CORPUS_DIR" ]]; then + echo "CV corpus directory not found: $CORPUS_DIR" >&2 + exit 1 +fi + +if ! find "$CORPUS_DIR" -maxdepth 1 -type f \( -iname '*.pdf' -o -iname '*.docx' -o -iname '*.txt' -o -iname '*.md' \) -print -quit | grep -q .; then + echo "No supported CV files were found in: $CORPUS_DIR" >&2 + exit 1 +fi mkdir -p "$OUTPUT_DIR" "$APPROVED_DIR" +echo "CV benchmark corpus: $CORPUS_DIR" echo "CV benchmark output: $OUTPUT_DIR" echo "Approved fixtures: $APPROVED_DIR" +CV_BENCHMARK_CORPUS_DIR="$CORPUS_DIR" \ CV_BENCHMARK_OUTPUT_DIR="$OUTPUT_DIR" \ CV_BENCHMARK_APPROVED_DIR="$APPROVED_DIR" \ dotnet test "$ROOT_DIR/JobTrackerApi.Tests/JobTrackerApi.Tests.csproj" --filter CvCorpusHarnessTests +test -s "$OUTPUT_DIR/index.json" +test -s "$OUTPUT_DIR/report.md" + if [[ -f "$OUTPUT_DIR/report.md" ]]; then echo echo "Benchmark report written to: $OUTPUT_DIR/report.md"