feat(ops): automate backups and private CV benchmarks
This commit is contained in:
cesnimda
@@ -98,6 +98,12 @@ MAINTENANCE_GITEA_BASE_URL=https://git.cesnimda.uk
|
||||
MAINTENANCE_GITEA_OWNER=cesnimda
|
||||
MAINTENANCE_GITEA_REPOSITORY=jobtrackingapp
|
||||
MAINTENANCE_GITEA_TOKEN=
|
||||
# Complete recovery sets are created before deployment and nightly at 02:00 Europe/Oslo.
|
||||
BACKUP_TIMEZONE=Europe/Oslo
|
||||
BACKUP_DAILY_RETENTION_DAYS=30
|
||||
BACKUP_MONTHLY_RETENTION_COUNT=12
|
||||
# Optional encrypted rclone destination, for example gdrive-crypt:jobtracker.
|
||||
BACKUP_RCLONE_REMOTE=
|
||||
EMAIL_ENABLED=false
|
||||
EMAIL_SMTP_HOST=smtp.gmail.com
|
||||
EMAIL_SMTP_PORT=587
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
name: Production backup
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# One of these UTC hours is 02:00 in Europe/Oslo; the guard skips the other.
|
||||
- cron: '0 0,1 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
backup:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Create verified production recovery set
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
with:
|
||||
host: ${{ secrets.PROD_HOST }}
|
||||
username: ${{ secrets.PROD_USER }}
|
||||
key: ${{ secrets.PROD_SSH_KEY }}
|
||||
command_timeout: 20m
|
||||
script: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "schedule" ] \
|
||||
&& [ "$(TZ=Europe/Oslo date +%H)" != "02" ]; then
|
||||
echo "Outside the 02:00 Europe/Oslo backup window."
|
||||
exit 0
|
||||
fi
|
||||
cd /opt/job-tracker/app
|
||||
if [ ! -x deploy/deploy.sh ]; then
|
||||
echo "The deployed backup script is unavailable."
|
||||
exit 1
|
||||
fi
|
||||
DEPLOY_BACKUP_ONLY=true deploy/deploy.sh
|
||||
@@ -15,28 +15,48 @@ jobs:
|
||||
if: inputs.job == 'cv-benchmark'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Setup .NET
|
||||
uses: actions/setup-dotnet@v4
|
||||
- name: Run private corpus benchmark on production
|
||||
uses: appleboy/ssh-action@v1.0.3
|
||||
with:
|
||||
dotnet-version: 9.0.317
|
||||
host: ${{ secrets.PROD_HOST }}
|
||||
username: ${{ secrets.PROD_USER }}
|
||||
key: ${{ secrets.PROD_SSH_KEY }}
|
||||
command_timeout: 30m
|
||||
script: |
|
||||
set -euo pipefail
|
||||
cd /opt/job-tracker/app
|
||||
corpus_dir=/home/pi/cvs
|
||||
approved_dir="$corpus_dir/approved-jsons"
|
||||
output_dir="$(mktemp -d)/latest"
|
||||
cleanup() { rm -rf "$(dirname "$output_dir")"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
- name: Run CV corpus benchmark
|
||||
env:
|
||||
CV_BENCHMARK_OUTPUT_DIR: ${{ github.workspace }}/tmp/cv-benchmarks/latest
|
||||
CV_BENCHMARK_APPROVED_DIR: ${{ github.workspace }}/tmp/cv-benchmarks/approved-fixtures
|
||||
run: |
|
||||
chmod +x scripts/run-cv-benchmark.sh
|
||||
scripts/run-cv-benchmark.sh
|
||||
test -s "$CV_BENCHMARK_OUTPUT_DIR/index.json"
|
||||
test -s "$CV_BENCHMARK_OUTPUT_DIR/report.md"
|
||||
AI_SERVICE_TOKEN="$(sed -n 's/^AI_SERVICE_TOKEN=//p' /opt/job-tracker/shared/.env | tail -n 1)"
|
||||
if [ -z "$AI_SERVICE_TOKEN" ]; then
|
||||
echo "AI_SERVICE_TOKEN is not configured." >&2
|
||||
exit 1
|
||||
fi
|
||||
ai_id="$(docker compose -f docker-compose.yml ps -q ai-service)"
|
||||
ai_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$ai_id")"
|
||||
if [ -z "$ai_ip" ]; then
|
||||
echo "The AI extraction service is not running." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Retain benchmark report and fixture candidates
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: cv-benchmark-${{ github.run_number }}
|
||||
path: tmp/cv-benchmarks
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
chmod +x scripts/run-cv-benchmark.sh
|
||||
CV_BENCHMARK_CORPUS_DIR="$corpus_dir" \
|
||||
CV_BENCHMARK_OUTPUT_DIR="$output_dir" \
|
||||
CV_BENCHMARK_APPROVED_DIR="$approved_dir" \
|
||||
CV_AI_BASE_URL="http://$ai_ip:8001" \
|
||||
CV_AI_SERVICE_TOKEN="$AI_SERVICE_TOKEN" \
|
||||
scripts/run-cv-benchmark.sh
|
||||
|
||||
backend_id="$(docker compose -f docker-compose.yml ps -q backend)"
|
||||
if [ -z "$backend_id" ]; then
|
||||
echo "The backend container is not running." >&2
|
||||
exit 1
|
||||
fi
|
||||
docker exec "$backend_id" sh -c 'mkdir -p /data/CvBenchmarks'
|
||||
docker cp "$output_dir/." "$backend_id:/data/CvBenchmarks/"
|
||||
docker exec "$backend_id" test -s /data/CvBenchmarks/index.json
|
||||
docker exec "$backend_id" test -s /data/CvBenchmarks/report.md
|
||||
|
||||
@@ -11,6 +11,7 @@ using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
@@ -52,24 +53,18 @@ public sealed class CvCorpusHarnessTests
|
||||
userManager.Setup(x => x.GetUserAsync(It.IsAny<System.Security.Claims.ClaimsPrincipal>())).ReturnsAsync(user);
|
||||
userManager.Setup(x => x.UpdateAsync(It.IsAny<ApplicationUser>())).ReturnsAsync(IdentityResult.Success);
|
||||
|
||||
var aiService = new Mock<ISummarizerService>();
|
||||
aiService.Setup(x => x.SummarizeSectionAsync(It.Is<string>(instruction => instruction.Contains("Extract this CV into structured JSON", StringComparison.Ordinal)), It.IsAny<string>(), 3200, 900)).ReturnsAsync(string.Empty);
|
||||
aiService.Setup(x => x.SummarizeSectionAsync(It.Is<string>(instruction => instruction.Contains("Reconstruct this CV text extracted from a PDF", StringComparison.Ordinal)), It.IsAny<string>(), 2800, 900)).ReturnsAsync((string _, string text, int _, int __) => text);
|
||||
var aiService = CreateSummarizerFromEnvironment();
|
||||
var cvAiNormalizer = CreateCvAiNormalizerFromEnvironment();
|
||||
|
||||
await using var db = TestHostFactory.CreateInMemoryDb();
|
||||
var paths = CreatePaths(outputRoot);
|
||||
var controller = new ProfileCvController(userManager.Object, aiService.Object, db, paths, null, NoOpCvAiClassifier.Instance, cvAiNormalizer)
|
||||
var controller = new ProfileCvController(userManager.Object, aiService, db, paths, null, NoOpCvAiClassifier.Instance, cvAiNormalizer)
|
||||
{
|
||||
ControllerContext = new ControllerContext { HttpContext = new DefaultHttpContext() }
|
||||
};
|
||||
|
||||
var extractMethod = typeof(ProfileCvController).GetMethod("ExtractTextAsync", BindingFlags.NonPublic | BindingFlags.Static);
|
||||
var reconstructMethod = typeof(ProfileCvController).GetMethod("MaybeReconstructStructuredCvAsync", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
var buildMethod = typeof(ProfileCvController).GetMethod("BuildStructuredCvAsync", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
var extractMethod = typeof(ProfileCvController).GetMethod("ExtractStructuredCvFromFileAsync", BindingFlags.NonPublic | BindingFlags.Instance);
|
||||
Assert.NotNull(extractMethod);
|
||||
Assert.NotNull(reconstructMethod);
|
||||
Assert.NotNull(buildMethod);
|
||||
|
||||
var entries = new List<CvBenchmarkEntry>();
|
||||
foreach (var path in files)
|
||||
@@ -83,16 +78,14 @@ public sealed class CvCorpusHarnessTests
|
||||
};
|
||||
|
||||
var extension = Path.GetExtension(path);
|
||||
var extractTask = (Task<string>)extractMethod!.Invoke(null, new object[] { formFile, extension })!;
|
||||
var text = await extractTask;
|
||||
var extractTask = (Task)extractMethod!.Invoke(controller, new object[] { formFile, extension, CancellationToken.None })!;
|
||||
await extractTask;
|
||||
var extraction = extractTask.GetType().GetProperty("Result")!.GetValue(extractTask)!;
|
||||
var resultType = extraction.GetType();
|
||||
var text = (string)resultType.GetProperty("RawText")!.GetValue(extraction)!;
|
||||
Assert.False(string.IsNullOrWhiteSpace(text));
|
||||
|
||||
var reconstructTask = (Task<string>)reconstructMethod!.Invoke(controller, new object[] { text, CancellationToken.None })!;
|
||||
var normalizedText = await reconstructTask;
|
||||
Assert.False(string.IsNullOrWhiteSpace(normalizedText));
|
||||
|
||||
var buildTask = (Task<StructuredCvProfile>)buildMethod!.Invoke(controller, new object[] { normalizedText, CancellationToken.None })!;
|
||||
var structured = StructuredCvProfileJson.Normalize(await buildTask);
|
||||
var structured = StructuredCvProfileJson.Normalize(
|
||||
(StructuredCvProfile)resultType.GetProperty("StructuredCv")!.GetValue(extraction)!);
|
||||
Assert.NotNull(structured);
|
||||
|
||||
var slug = Slugify(fileName);
|
||||
@@ -382,9 +375,44 @@ public sealed class CvCorpusHarnessTests
|
||||
{
|
||||
client.BaseAddress = new Uri(baseUrl.Trim());
|
||||
client.Timeout = TimeSpan.FromSeconds(180);
|
||||
var token = Environment.GetEnvironmentVariable("CV_AI_SERVICE_TOKEN");
|
||||
if (!string.IsNullOrWhiteSpace(token)) client.DefaultRequestHeaders.Add("X-Ai-Service-Token", token);
|
||||
});
|
||||
var provider = services.BuildServiceProvider();
|
||||
var factory = provider.GetRequiredService<System.Net.Http.IHttpClientFactory>();
|
||||
return new CvAiNormalizer(factory);
|
||||
}
|
||||
|
||||
private static ISummarizerService CreateSummarizerFromEnvironment()
|
||||
{
|
||||
var baseUrl = Environment.GetEnvironmentVariable("CV_AI_BASE_URL");
|
||||
if (string.IsNullOrWhiteSpace(baseUrl))
|
||||
{
|
||||
var mock = new Mock<ISummarizerService>();
|
||||
mock.Setup(x => x.ExtractTextAsync(It.IsAny<Stream>(), It.IsAny<string>(), It.IsAny<string?>(), It.IsAny<CancellationToken>()))
|
||||
.Returns(async (Stream stream, string fileName, string? contentType, CancellationToken cancellationToken) =>
|
||||
{
|
||||
using var reader = new StreamReader(stream);
|
||||
var text = await reader.ReadToEndAsync(cancellationToken);
|
||||
return new AiTextExtractionResult(text, false, contentType ?? "text/plain", null, text.Length, fileName);
|
||||
});
|
||||
mock.Setup(x => x.SummarizeSectionAsync(It.Is<string>(instruction => instruction.Contains("Extract this CV into structured JSON", StringComparison.Ordinal)), It.IsAny<string>(), 3200, 900)).ReturnsAsync(string.Empty);
|
||||
mock.Setup(x => x.SummarizeSectionAsync(It.Is<string>(instruction => instruction.Contains("Reconstruct this CV text extracted from a PDF", StringComparison.Ordinal)), It.IsAny<string>(), 2800, 900)).ReturnsAsync((string _, string text, int _, int __) => text);
|
||||
return mock.Object;
|
||||
}
|
||||
|
||||
var services = new Microsoft.Extensions.DependencyInjection.ServiceCollection();
|
||||
services.AddMemoryCache();
|
||||
services.AddHttpClient("ai-service", client =>
|
||||
{
|
||||
client.BaseAddress = new Uri(baseUrl.Trim());
|
||||
client.Timeout = TimeSpan.FromMinutes(3);
|
||||
var token = Environment.GetEnvironmentVariable("CV_AI_SERVICE_TOKEN");
|
||||
if (!string.IsNullOrWhiteSpace(token)) client.DefaultRequestHeaders.Add("X-Ai-Service-Token", token);
|
||||
});
|
||||
var provider = services.BuildServiceProvider();
|
||||
return new SummarizerService(
|
||||
provider.GetRequiredService<System.Net.Http.IHttpClientFactory>(),
|
||||
provider.GetRequiredService<IMemoryCache>());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
Alex Example
|
||||
Software Developer
|
||||
alex.example@example.test | +44 20 7946 0000 | London, United Kingdom
|
||||
https://example.test/alex
|
||||
|
||||
Professional Summary
|
||||
Software developer building reliable web applications and APIs.
|
||||
|
||||
Technical Skills
|
||||
C#, .NET, TypeScript, SQL, Docker
|
||||
|
||||
Professional Experience
|
||||
Software Developer | Example Systems Ltd | London, United Kingdom
|
||||
January 2022 - Present
|
||||
- Built accessible web application features using .NET and TypeScript.
|
||||
- Improved automated test coverage for critical application workflows.
|
||||
|
||||
Education
|
||||
BSc Computer Science | Example University | London, United Kingdom
|
||||
September 2018 - June 2021
|
||||
|
||||
Languages
|
||||
English - Native
|
||||
Norwegian Bokmal - Intermediate
|
||||
@@ -17,7 +17,7 @@ public sealed class MaintenanceJobDispatcher(IConfiguration configuration, IHttp
|
||||
new(
|
||||
CvBenchmarkJobId,
|
||||
"CV extraction benchmark",
|
||||
"Runs the CV corpus benchmark and retains the report and fixture candidates as a workflow artifact.",
|
||||
"Runs the private CV corpus benchmark on the production host and publishes its report to Admin/System.",
|
||||
settings.IsConfigured,
|
||||
settings.RunsUrl)
|
||||
];
|
||||
|
||||
+7
-3
@@ -236,9 +236,13 @@ MYSQL_PWD='<password>' mariadb-dump \
|
||||
|
||||
### Retention
|
||||
|
||||
**Nothing is deleted automatically.** Backups accumulate in `BACKUP_DIR` until you remove them. Watch
|
||||
disk usage and prune deliberately — a suggested policy is to keep every backup for 30 days and one per
|
||||
month after that, but the script does not enforce it and will not delete your files.
|
||||
Verified top-level recovery sets are retained for 30 days and the first complete set in each month is
|
||||
copied into `monthly/`; the newest 12 monthly sets are retained. Configure
|
||||
`BACKUP_DAILY_RETENTION_DAYS` and `BACKUP_MONTHLY_RETENTION_COUNT` to change those positive integer
|
||||
limits. Paths outside the fixed backup directory and monthly naming contract are never pruned.
|
||||
|
||||
Set `BACKUP_RCLONE_REMOTE` to an encrypted rclone remote to copy each current recovery set off-host.
|
||||
See [`docs/operations/google-drive-backups.md`](../docs/operations/google-drive-backups.md).
|
||||
|
||||
## Restoring the database
|
||||
|
||||
|
||||
+79
-2
@@ -265,7 +265,7 @@ validate_deploy_config() {
|
||||
# exactly when a restore point matters. See deploy/README.md.
|
||||
# ---------------------------------------------------------------------------
|
||||
BACKUP_DIR="${BACKUP_DIR:-/opt/job-tracker/backups}"
|
||||
BACKUP_STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
BACKUP_STAMP="${DEPLOY_BACKUP_STAMP:-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
|
||||
backup_database() {
|
||||
if [ "${DEPLOY_SKIP_DB_BACKUP:-false}" = "true" ]; then
|
||||
@@ -406,9 +406,9 @@ verify_backup_file_shape() {
|
||||
|
||||
report_backup() {
|
||||
local target="$1" size
|
||||
chmod 600 "$target"
|
||||
size="$(du -h "$target" | cut -f1)"
|
||||
echo "Backup verified: ${target} (${size})"
|
||||
echo "Retention is manual — old backups in ${BACKUP_DIR} are never deleted automatically."
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -513,6 +513,74 @@ backup_recovery_volumes() {
|
||||
"deletion-tombstones-${project_name}" false
|
||||
}
|
||||
|
||||
apply_backup_retention() {
|
||||
local daily_days="${BACKUP_DAILY_RETENTION_DAYS:-30}"
|
||||
local monthly_count="${BACKUP_MONTHLY_RETENTION_COUNT:-12}"
|
||||
local timezone="${BACKUP_TIMEZONE:-Europe/Oslo}"
|
||||
if [[ ! "$daily_days" =~ ^[1-9][0-9]*$ ]] || [[ ! "$monthly_count" =~ ^[1-9][0-9]*$ ]]; then
|
||||
echo "Backup retention must use positive integer day/count values."
|
||||
return 1
|
||||
fi
|
||||
|
||||
local month monthly_root monthly_dir
|
||||
month="$(TZ="$timezone" date +%Y-%m)"
|
||||
monthly_root="$BACKUP_DIR/monthly"
|
||||
monthly_dir="$monthly_root/$month"
|
||||
mkdir -p "$monthly_root"
|
||||
chmod 700 "$monthly_root"
|
||||
|
||||
local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*)
|
||||
if [ ! -e "${current_files[0]}" ]; then
|
||||
echo "No verified backup files were found for stamp ${BACKUP_STAMP}."
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ ! -f "$monthly_dir/.complete" ]; then
|
||||
mkdir -p "$monthly_dir"
|
||||
chmod 700 "$monthly_dir"
|
||||
cp --preserve=mode,timestamps -- "${current_files[@]}" "$monthly_dir/"
|
||||
printf '%s\n' "$BACKUP_STAMP" > "$monthly_dir/.complete"
|
||||
chmod 600 "$monthly_dir/.complete"
|
||||
echo "Monthly recovery set retained: ${monthly_dir}"
|
||||
fi
|
||||
|
||||
find "$BACKUP_DIR" -maxdepth 1 -type f -name 'jobtracker-*' -mtime "+$daily_days" -delete
|
||||
|
||||
local monthly_dirs=()
|
||||
mapfile -t monthly_dirs < <(find "$monthly_root" -mindepth 1 -maxdepth 1 -type d -name '20??-??' | sort)
|
||||
while [ "${#monthly_dirs[@]}" -gt "$monthly_count" ]; do
|
||||
local oldest="${monthly_dirs[0]}"
|
||||
case "$oldest" in
|
||||
"$monthly_root"/20??-??) rm -rf -- "$oldest" ;;
|
||||
*) echo "Refusing to prune unexpected monthly backup path: $oldest"; return 1 ;;
|
||||
esac
|
||||
monthly_dirs=("${monthly_dirs[@]:1}")
|
||||
done
|
||||
|
||||
echo "Backup retention applied: ${daily_days} days daily, ${monthly_count} monthly sets."
|
||||
}
|
||||
|
||||
upload_current_backup_set() {
|
||||
local remote="${BACKUP_RCLONE_REMOTE:-}"
|
||||
[ -n "$remote" ] || return 0
|
||||
if ! command -v rclone >/dev/null 2>&1; then
|
||||
echo "BACKUP_RCLONE_REMOTE is configured but rclone is not installed."
|
||||
return 1
|
||||
fi
|
||||
|
||||
remote="${remote%/}"
|
||||
local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*) file
|
||||
for file in "${current_files[@]}"; do
|
||||
rclone copyto "$file" "$remote/daily/$(basename "$file")" --immutable
|
||||
done
|
||||
|
||||
local month="$(TZ="${BACKUP_TIMEZONE:-Europe/Oslo}" date +%Y-%m)"
|
||||
if [ -d "$BACKUP_DIR/monthly/$month" ]; then
|
||||
rclone copy "$BACKUP_DIR/monthly/$month" "$remote/monthly/$month" --immutable
|
||||
fi
|
||||
echo "Current recovery set copied to configured encrypted off-host remote."
|
||||
}
|
||||
|
||||
resolve_existing_web_proxy_subnet
|
||||
|
||||
if ! validate_deploy_config; then
|
||||
@@ -527,6 +595,15 @@ if ! backup_recovery_volumes; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! apply_backup_retention || ! upload_current_backup_set; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${DEPLOY_BACKUP_ONLY:-false}" = "true" ]; then
|
||||
echo "Backup-only run complete: ${BACKUP_STAMP}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
previous_backend_ref=""
|
||||
previous_frontend_ref=""
|
||||
capture_previous_core_image() {
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# Encrypted Google Drive backups
|
||||
|
||||
Yes—JobTracker recovery sets can be copied to Google Drive. Use an `rclone crypt` remote so database,
|
||||
CV, attachment and key archives are encrypted before they leave the production host.
|
||||
|
||||
## One-time production setup
|
||||
|
||||
1. Install `rclone` from the operating system's trusted package source.
|
||||
2. Run `rclone config` as the same operating-system user used by deployment.
|
||||
3. Create a Google Drive remote, for example `gdrive`.
|
||||
4. Complete Google OAuth in your browser. Do not paste its refresh token into Git, chat or CI.
|
||||
5. Create a second remote of type `crypt`, for example `gdrive-crypt`, targeting a dedicated folder
|
||||
such as `gdrive:JobTrackerBackups`.
|
||||
6. Enable filename and directory-name encryption. Store the crypt password and salt in the protected
|
||||
production credential store/rclone configuration, not in the repository.
|
||||
7. Verify the encrypted remote without uploading private application data:
|
||||
|
||||
```bash
|
||||
printf 'backup connectivity test\n' >/tmp/jobtracker-backup-test.txt
|
||||
rclone copy /tmp/jobtracker-backup-test.txt gdrive-crypt:jobtracker-test
|
||||
rclone ls gdrive-crypt:jobtracker-test
|
||||
rm -f /tmp/jobtracker-backup-test.txt
|
||||
rclone purge gdrive-crypt:jobtracker-test
|
||||
```
|
||||
|
||||
8. Add this to `/opt/job-tracker/shared/.env`:
|
||||
|
||||
```text
|
||||
BACKUP_RCLONE_REMOTE=gdrive-crypt:jobtracker
|
||||
```
|
||||
|
||||
9. Run the **Production backup** workflow manually. It must create the local verified recovery set
|
||||
first, copy that set to Drive, and fail visibly if the off-host copy fails.
|
||||
|
||||
## Schedule and retention
|
||||
|
||||
- Scheduled workflow: 00:00 and 01:00 UTC triggers with a Europe/Oslo guard; one backup runs at local
|
||||
02:00 even when daylight-saving time changes.
|
||||
- Recovery-point objective: 24 hours.
|
||||
- Local retention: 30 days plus 12 monthly recovery sets.
|
||||
- Google Drive retention: the upload is immutable. Apply a matching Drive/rclone lifecycle only after
|
||||
a scratch restore has proven the off-host copy.
|
||||
|
||||
## Recovery rule
|
||||
|
||||
Download into a scratch directory and restore into a separate MariaDB/database-volume environment.
|
||||
Never rehearse by restoring over live production. Preserve the newest deletion-tombstone ledger when
|
||||
restoring older application data.
|
||||
@@ -4,16 +4,32 @@ set -euo pipefail
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
OUTPUT_DIR="${CV_BENCHMARK_OUTPUT_DIR:-$ROOT_DIR/tmp/cv-benchmarks/latest}"
|
||||
APPROVED_DIR="${CV_BENCHMARK_APPROVED_DIR:-$ROOT_DIR/tmp/cv-benchmarks/approved-fixtures}"
|
||||
CORPUS_DIR="${CV_BENCHMARK_CORPUS_DIR:-/home/pi/cvs}"
|
||||
|
||||
if [[ ! -d "$CORPUS_DIR" ]]; then
|
||||
echo "CV corpus directory not found: $CORPUS_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! find "$CORPUS_DIR" -maxdepth 1 -type f \( -iname '*.pdf' -o -iname '*.docx' -o -iname '*.txt' -o -iname '*.md' \) -print -quit | grep -q .; then
|
||||
echo "No supported CV files were found in: $CORPUS_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$OUTPUT_DIR" "$APPROVED_DIR"
|
||||
|
||||
echo "CV benchmark corpus: $CORPUS_DIR"
|
||||
echo "CV benchmark output: $OUTPUT_DIR"
|
||||
echo "Approved fixtures: $APPROVED_DIR"
|
||||
|
||||
CV_BENCHMARK_CORPUS_DIR="$CORPUS_DIR" \
|
||||
CV_BENCHMARK_OUTPUT_DIR="$OUTPUT_DIR" \
|
||||
CV_BENCHMARK_APPROVED_DIR="$APPROVED_DIR" \
|
||||
dotnet test "$ROOT_DIR/JobTrackerApi.Tests/JobTrackerApi.Tests.csproj" --filter CvCorpusHarnessTests
|
||||
|
||||
test -s "$OUTPUT_DIR/index.json"
|
||||
test -s "$OUTPUT_DIR/report.md"
|
||||
|
||||
if [[ -f "$OUTPUT_DIR/report.md" ]]; then
|
||||
echo
|
||||
echo "Benchmark report written to: $OUTPUT_DIR/report.md"
|
||||
|
||||
Reference in New Issue
Block a user