feat(ops): automate backups and private CV benchmarks

deploy/README.md

@@ -236,9 +236,13 @@ MYSQL_PWD='<password>' mariadb-dump \
 
 ### Retention
 
-**Nothing is deleted automatically.** Backups accumulate in `BACKUP_DIR` until you remove them. Watch
-disk usage and prune deliberately — a suggested policy is to keep every backup for 30 days and one per
-month after that, but the script does not enforce it and will not delete your files.
+Verified top-level recovery sets are retained for 30 days and the first complete set in each month is
+copied into `monthly/`; the newest 12 monthly sets are retained. Configure
+`BACKUP_DAILY_RETENTION_DAYS` and `BACKUP_MONTHLY_RETENTION_COUNT` to change those positive integer
+limits. Paths outside the fixed backup directory and monthly naming contract are never pruned.
+
+Set `BACKUP_RCLONE_REMOTE` to an encrypted rclone remote to copy each current recovery set off-host.
+See [`docs/operations/google-drive-backups.md`](../docs/operations/google-drive-backups.md).
 
 ## Restoring the database
 

deploy/deploy.sh

@@ -265,7 +265,7 @@ validate_deploy_config() {
 # exactly when a restore point matters. See deploy/README.md.
 # ---------------------------------------------------------------------------
 BACKUP_DIR="${BACKUP_DIR:-/opt/job-tracker/backups}"
-BACKUP_STAMP="$(date -u +%Y%m%dT%H%M%SZ)"
+BACKUP_STAMP="${DEPLOY_BACKUP_STAMP:-$(date -u +%Y%m%dT%H%M%SZ)}"
 
 backup_database() {
   if [ "${DEPLOY_SKIP_DB_BACKUP:-false}" = "true" ]; then
@@ -406,9 +406,9 @@ verify_backup_file_shape() {
 
 report_backup() {
   local target="$1" size
+  chmod 600 "$target"
   size="$(du -h "$target" | cut -f1)"
   echo "Backup verified: ${target} (${size})"
-  echo "Retention is manual — old backups in ${BACKUP_DIR} are never deleted automatically."
   return 0
 }
 
@@ -513,6 +513,74 @@ backup_recovery_volumes() {
     "deletion-tombstones-${project_name}" false
 }
 
+apply_backup_retention() {
+  local daily_days="${BACKUP_DAILY_RETENTION_DAYS:-30}"
+  local monthly_count="${BACKUP_MONTHLY_RETENTION_COUNT:-12}"
+  local timezone="${BACKUP_TIMEZONE:-Europe/Oslo}"
+  if [[ ! "$daily_days" =~ ^[1-9][0-9]*$ ]] || [[ ! "$monthly_count" =~ ^[1-9][0-9]*$ ]]; then
+    echo "Backup retention must use positive integer day/count values."
+    return 1
+  fi
+
+  local month monthly_root monthly_dir
+  month="$(TZ="$timezone" date +%Y-%m)"
+  monthly_root="$BACKUP_DIR/monthly"
+  monthly_dir="$monthly_root/$month"
+  mkdir -p "$monthly_root"
+  chmod 700 "$monthly_root"
+
+  local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*)
+  if [ ! -e "${current_files[0]}" ]; then
+    echo "No verified backup files were found for stamp ${BACKUP_STAMP}."
+    return 1
+  fi
+
+  if [ ! -f "$monthly_dir/.complete" ]; then
+    mkdir -p "$monthly_dir"
+    chmod 700 "$monthly_dir"
+    cp --preserve=mode,timestamps -- "${current_files[@]}" "$monthly_dir/"
+    printf '%s\n' "$BACKUP_STAMP" > "$monthly_dir/.complete"
+    chmod 600 "$monthly_dir/.complete"
+    echo "Monthly recovery set retained: ${monthly_dir}"
+  fi
+
+  find "$BACKUP_DIR" -maxdepth 1 -type f -name 'jobtracker-*' -mtime "+$daily_days" -delete
+
+  local monthly_dirs=()
+  mapfile -t monthly_dirs < <(find "$monthly_root" -mindepth 1 -maxdepth 1 -type d -name '20??-??' | sort)
+  while [ "${#monthly_dirs[@]}" -gt "$monthly_count" ]; do
+    local oldest="${monthly_dirs[0]}"
+    case "$oldest" in
+      "$monthly_root"/20??-??) rm -rf -- "$oldest" ;;
+      *) echo "Refusing to prune unexpected monthly backup path: $oldest"; return 1 ;;
+    esac
+    monthly_dirs=("${monthly_dirs[@]:1}")
+  done
+
+  echo "Backup retention applied: ${daily_days} days daily, ${monthly_count} monthly sets."
+}
+
+upload_current_backup_set() {
+  local remote="${BACKUP_RCLONE_REMOTE:-}"
+  [ -n "$remote" ] || return 0
+  if ! command -v rclone >/dev/null 2>&1; then
+    echo "BACKUP_RCLONE_REMOTE is configured but rclone is not installed."
+    return 1
+  fi
+
+  remote="${remote%/}"
+  local current_files=("$BACKUP_DIR"/jobtracker-*"-$BACKUP_STAMP".*) file
+  for file in "${current_files[@]}"; do
+    rclone copyto "$file" "$remote/daily/$(basename "$file")" --immutable
+  done
+
+  local month="$(TZ="${BACKUP_TIMEZONE:-Europe/Oslo}" date +%Y-%m)"
+  if [ -d "$BACKUP_DIR/monthly/$month" ]; then
+    rclone copy "$BACKUP_DIR/monthly/$month" "$remote/monthly/$month" --immutable
+  fi
+  echo "Current recovery set copied to configured encrypted off-host remote."
+}
+
 resolve_existing_web_proxy_subnet
 
 if ! validate_deploy_config; then
@@ -527,6 +595,15 @@ if ! backup_recovery_volumes; then
   exit 1
 fi
 
+if ! apply_backup_retention || ! upload_current_backup_set; then
+  exit 1
+fi
+
+if [ "${DEPLOY_BACKUP_ONLY:-false}" = "true" ]; then
+  echo "Backup-only run complete: ${BACKUP_STAMP}"
+  exit 0
+fi
+
 previous_backend_ref=""
 previous_frontend_ref=""
 capture_previous_core_image() {