#!/usr/bin/env bash set -euo pipefail cd "$(dirname "$0")/.." ENV_SOURCE="/opt/job-tracker/shared/.env" ENV_TARGET=".env" if [ ! -f "$ENV_SOURCE" ]; then echo "Missing shared env file at $ENV_SOURCE" exit 1 fi # Keep runtime secrets outside the repo checkout so workflow uploads cannot wipe them. ln -snf "$ENV_SOURCE" "$ENV_TARGET" if [ ! -L "$ENV_TARGET" ] && [ ! -f "$ENV_TARGET" ]; then echo "Failed to link deployment env file into $(pwd)/$ENV_TARGET" exit 1 fi export APP_VERSION="${APP_VERSION:-0.0.0}" export APP_COMMIT_SHA="${APP_COMMIT_SHA:-unknown}" export APP_BUILD_STAMP="${APP_BUILD_STAMP:-unknown}" export DEPLOY_BUILD_AI_SERVICE="${DEPLOY_BUILD_AI_SERVICE:-false}" compose() { docker compose "$@" } # --------------------------------------------------------------------------- # Database backup, taken BEFORE anything is stopped, built or replaced. # # A failed backup fails the deploy. The startup reconciler is additive and never # drops a table holding rows, but "additive" is a property of the code, not a # guarantee about the disk — and the first deploy after a schema change is # exactly when a restore point matters. See deploy/README.md. # --------------------------------------------------------------------------- BACKUP_DIR="${BACKUP_DIR:-/opt/job-tracker/backups}" backup_database() { if [ "${DEPLOY_SKIP_DB_BACKUP:-false}" = "true" ]; then echo "WARNING: DEPLOY_SKIP_DB_BACKUP=true — deploying with no restore point. Emergency use only." return 0 fi local provider="${DATABASE_PROVIDER:-sqlite}" local stamp stamp="$(date -u +%Y%m%dT%H%M%SZ)" mkdir -p "$BACKUP_DIR" if [ "$provider" != "mysql" ] && [ "$provider" != "mariadb" ]; then # SQLite lives in the jobtracker_data volume. Tar it from a throwaway container # so the host needs no sqlite tooling and no knowledge of the volume layout. local target="$BACKUP_DIR/jobtracker-sqlite-${stamp}.tar.gz" echo "Backing up SQLite data volume to ${target}" # The archive path is built inside the container: passing /backup/... as an argument # gets rewritten by MSYS path translation when the script is run from Git Bash. if ! docker run --rm \ -v jobtracker_data:/data:ro \ -v "$BACKUP_DIR":/backup \ -e ARCHIVE_NAME="$(basename "$target")" \ alpine:3 sh -c 'tar czf "/backup/$ARCHIVE_NAME" -C /data .'; then echo "SQLite volume backup FAILED. Aborting deploy." return 1 fi verify_backup "$target" return $? fi local cs="${JOBTRACKER_CONNECTION_STRING:-}" if [ -z "$cs" ]; then echo "DATABASE_PROVIDER=${provider} but JOBTRACKER_CONNECTION_STRING is empty. Aborting deploy." return 1 fi # Parse the ADO.NET connection string. Keys are case-insensitive in .NET, so match that. local db_host db_port db_name db_user db_pass db_host="$(sed -n 's/.*[Ss]erver=\([^;]*\).*/\1/p' <<<"$cs")" db_port="$(sed -n 's/.*[Pp]ort=\([^;]*\).*/\1/p' <<<"$cs")" db_name="$(sed -n 's/.*[Dd]atabase=\([^;]*\).*/\1/p' <<<"$cs")" db_user="$(sed -n 's/.*[Uu]ser[ ]*[Ii]*[Dd]*=\([^;]*\).*/\1/p' <<<"$cs")" db_pass="$(sed -n 's/.*[Pp]assword=\([^;]*\).*/\1/p' <<<"$cs")" db_port="${db_port:-3306}" if [ -z "$db_host" ] || [ -z "$db_name" ] || [ -z "$db_user" ]; then echo "Could not parse host/database/user from JOBTRACKER_CONNECTION_STRING. Aborting deploy." return 1 fi local target="$BACKUP_DIR/jobtracker-${db_name}-${stamp}.sql.gz" echo "Backing up MariaDB database '${db_name}' on ${db_host}:${db_port} to ${target}" # The password goes via MYSQL_PWD, never on the command line, so it cannot leak # into the process list or the deploy log. local dump_status=0 if command -v mariadb-dump >/dev/null 2>&1; then MYSQL_PWD="$db_pass" mariadb-dump \ --host="$db_host" --port="$db_port" --user="$db_user" \ --single-transaction --routines --events --quick \ "$db_name" 2>/tmp/jobtracker-backup.err | gzip -c > "$target" || dump_status=$? elif command -v mysqldump >/dev/null 2>&1; then MYSQL_PWD="$db_pass" mysqldump \ --host="$db_host" --port="$db_port" --user="$db_user" \ --single-transaction --routines --events --quick \ "$db_name" 2>/tmp/jobtracker-backup.err | gzip -c > "$target" || dump_status=$? else # No client on the host: run one in a container. --network host so the same # host/port from the connection string resolves identically. docker run --rm --network host -e MYSQL_PWD="$db_pass" mariadb:11 \ mariadb-dump --host="$db_host" --port="$db_port" --user="$db_user" \ --single-transaction --routines --events --quick \ "$db_name" 2>/tmp/jobtracker-backup.err | gzip -c > "$target" || dump_status=$? fi if [ "$dump_status" -ne 0 ]; then echo "Database dump FAILED (exit ${dump_status}). Aborting deploy." sed -e 's/password=[^ ]*/password=***/gI' /tmp/jobtracker-backup.err >&2 || true rm -f "$target" return 1 fi verify_backup "$target" "CREATE TABLE" } # A dump that exists but is empty or truncated is worse than none, because it # looks like a restore point. Check size, and content when we know what to expect. verify_backup() { local target="$1" local expect="${2:-}" if [ ! -s "$target" ]; then echo "Backup file ${target} is missing or empty. Aborting deploy." rm -f "$target" return 1 fi local size size="$(du -h "$target" | cut -f1)" if [ -n "$expect" ] && ! gzip -dc "$target" | grep -q "$expect"; then echo "Backup ${target} does not contain '${expect}' — it is not a usable dump. Aborting deploy." rm -f "$target" return 1 fi echo "Backup verified: ${target} (${size})" echo "Retention is manual — old backups in ${BACKUP_DIR} are never deleted automatically." return 0 } if ! backup_database; then exit 1 fi build_core_with_recovery() { if compose build backend frontend; then return 0 fi echo "docker compose build for core services failed. Attempting one cleanup + retry because layer extraction can fail on constrained hosts." docker builder prune -af >/dev/null 2>&1 || true docker system prune -f >/dev/null 2>&1 || true compose build --no-cache backend frontend } build_ai_with_recovery() { if compose build ai-service; then return 0 fi echo "docker compose build for ai-service failed. Attempting one cleanup + retry because layer extraction can fail on constrained hosts." docker image rm -f app-ai-service:latest 2>/dev/null || true docker builder prune -af >/dev/null 2>&1 || true docker system prune -f >/dev/null 2>&1 || true compose build --no-cache ai-service } compose pull || true build_core_with_recovery if [ "$DEPLOY_BUILD_AI_SERVICE" = "true" ]; then build_ai_with_recovery else echo "Skipping ai-service rebuild during deploy (set DEPLOY_BUILD_AI_SERVICE=true to rebuild it)." fi # Force recreation so updated port mappings, env vars, and container config always apply on deploy. compose up -d --force-recreate --remove-orphans backend frontend if [ "$DEPLOY_BUILD_AI_SERVICE" = "true" ]; then # Ollama is opt-in (compose "bundled-ollama" profile). Deploys reuse an # existing/shared Ollama via OLLAMA_BASE_URL instead of starting a duplicate. compose up -d --force-recreate ai-service fi if [ -n "${OLLAMA_MODEL:-}" ]; then echo "Post-deploy Ollama warmup enabled for model: ${OLLAMA_MODEL}" ./scripts/start-ollama-cv.sh fi sleep 5 compose ps backend_status="$(compose ps backend --format '{{.State}}' 2>/dev/null | head -n 1 | tr '[:upper:]' '[:lower:]')" if [ "$backend_status" != "running" ]; then echo "Backend service is not healthy after deploy (state: ${backend_status:-unknown})." compose logs --tail=200 backend || true exit 1 fi ai_status="$(compose ps ai-service --format '{{.State}}' 2>/dev/null | head -n 1 | tr '[:upper:]' '[:lower:]')" if [ "$ai_status" != "running" ]; then echo "AI service is not healthy after deploy (state: ${ai_status:-unknown}). Continuing because AI is not a deploy gate for the core app." compose logs --tail=200 ai-service || true fi if [ -n "${APP_PUBLIC_BASE_URL:-}" ]; then public_base="${APP_PUBLIC_BASE_URL%/}" auth_config_body_file="$(mktemp)" auth_config_headers_file="$(mktemp)" cleanup_public_check() { rm -f "$auth_config_body_file" "$auth_config_headers_file" } trap cleanup_public_check EXIT echo "Running public smoke check against ${public_base}" if ! curl -fsS "${public_base}/" >/dev/null; then echo "Public frontend check failed for ${public_base}/" exit 1 fi if ! curl -fsS -D "$auth_config_headers_file" -o "$auth_config_body_file" "${public_base}/api/auth/config"; then echo "Public API smoke check failed for ${public_base}/api/auth/config" exit 1 fi content_type="$(awk 'BEGIN{IGNORECASE=1} /^content-type:/ {print $2}' "$auth_config_headers_file" | tr -d '\r' | tail -n 1)" if [[ "$content_type" != application/json* ]]; then echo "Public API smoke check returned unexpected content type: ${content_type:-missing}" echo "First bytes of response:" head -c 200 "$auth_config_body_file" || true exit 1 fi if ! grep -q 'requireAuth' "$auth_config_body_file"; then echo "Public API smoke check returned JSON without requireAuth." cat "$auth_config_body_file" exit 1 fi trap - EXIT cleanup_public_check fi # Clean up old legacy container name if it still exists from pre-rename deployments. docker rm -f app-summarizer-1 2>/dev/null || true echo "Deployment complete: ${APP_VERSION} ${APP_COMMIT_SHA}"