jobtrackingapp/deploy/deploy.sh

#!/usr/bin/env bash
set -euo pipefail

cd "$(dirname "$0")/.."

ENV_SOURCE="/opt/job-tracker/shared/.env"
ENV_TARGET=".env"

if [ ! -f "$ENV_SOURCE" ]; then
  echo "Missing shared env file at $ENV_SOURCE"
  exit 1
fi

# Keep runtime secrets outside the repo checkout so workflow uploads cannot wipe them.
ln -snf "$ENV_SOURCE" "$ENV_TARGET"

if [ ! -L "$ENV_TARGET" ] && [ ! -f "$ENV_TARGET" ]; then
  echo "Failed to link deployment env file into $(pwd)/$ENV_TARGET"
  exit 1
fi

# ---------------------------------------------------------------------------
# Load the shared environment into THIS shell.
#
# The symlink above is for docker compose, which reads .env itself. The script's
# own decisions — which backup to take, which variables are missing — ran against
# an empty environment until this existed, so DATABASE_PROVIDER defaulted to
# sqlite and a MariaDB host silently got a volume tar instead of a dump, with a
# "Backup verified" line to match. See docs/release-candidate-review.md (B1).
#
# Parsed line by line rather than sourced: a compose .env is not a shell script,
# so an unquoted value containing spaces would execute as a command under `.`.
# Values already present in the environment win, so CI-provided APP_VERSION and
# friends still override the file. No value is ever echoed.
# ---------------------------------------------------------------------------
load_env_file() {
  local file="$1" line key value

  while IFS= read -r line || [ -n "$line" ]; do
    line="${line%$'\r'}"
    case "$line" in ''|'#'*) continue ;; esac
    case "$line" in *=*) ;; *) continue ;; esac

    key="${line%%=*}"
    value="${line#*=}"
    key="${key#export }"
    key="${key//[[:space:]]/}"
    case "$key" in ''|*[!A-Za-z0-9_]*) continue ;; esac

    case "$value" in
      \"*\") value="${value#\"}"; value="${value%\"}" ;;
      \'*\') value="${value#\'}"; value="${value%\'}" ;;
    esac

    # Already set (CI, or an explicit override on the command line) wins.
    [ -n "${!key+x}" ] && continue
    export "$key=$value"
  done < "$file"
}

load_env_file "$ENV_TARGET"
echo "Loaded deployment environment from ${ENV_SOURCE}"

export APP_VERSION="${APP_VERSION:-0.0.0}"
export APP_COMMIT_SHA="${APP_COMMIT_SHA:-unknown}"
export APP_BUILD_STAMP="${APP_BUILD_STAMP:-unknown}"
export DEPLOY_BUILD_AI_SERVICE="${DEPLOY_BUILD_AI_SERVICE:-false}"

compose() {
  docker compose "$@"
}

# ---------------------------------------------------------------------------
# Configuration validation, before anything is built, stopped or replaced.
#
# Everything checked here is fatal later anyway — compose declares AI_SERVICE_TOKEN
# with `:?`, and the backend throws on a blank Auth__JwtKey. Failing here costs an
# aborted deploy; failing there costs a half-replaced stack.
#
# Names only in the output. Never values.
# ---------------------------------------------------------------------------
require_var() {
  local name="$1" hint="$2"

  if [ -z "${!name:-}" ]; then
    echo "Missing required deployment variable: ${name}"
    echo "    ${hint}"
    echo "    Set it in ${ENV_SOURCE}."
    return 1
  fi
  return 0
}

validate_deploy_config() {
  local failed=0

  # Deliberately no default. Guessing this wrong means backing up the wrong
  # database and reporting success — the exact failure this block exists to stop.
  if [ -z "${DATABASE_PROVIDER:-}" ]; then
    echo "Missing required deployment variable: DATABASE_PROVIDER"
    echo "    Set to 'mariadb' (or 'mysql') for a MariaDB deployment, or 'sqlite'."
    echo "    There is no default: the wrong value backs up the wrong database."
    echo "    Set it in ${ENV_SOURCE}."
    failed=1
  else
    case "$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')" in
      mysql|mariadb)
        require_var JOBTRACKER_CONNECTION_STRING \
          "MariaDB connection string. Required to take a database dump." || failed=1
        ;;
      sqlite)
        ;;
      *)
        echo "DATABASE_PROVIDER is not a recognised value. Use mariadb, mysql or sqlite."
        failed=1
        ;;
    esac
  fi

  require_var AI_SERVICE_TOKEN \
    "Shared secret between backend and ai-service. docker compose refuses to start without it." || failed=1

  # docker-compose.yml sets Auth__Require=true unconditionally, and the backend
  # throws at startup rather than issuing tokens signed with a blank key.
  require_var AUTH_JWT_KEY \
    "JWT signing key. With Auth__Require=true the backend refuses to start without it." || failed=1

  if [ -z "${APP_PUBLIC_BASE_URL:-}" ]; then
    echo "Note: APP_PUBLIC_BASE_URL is not set — the post-deploy public smoke check will be skipped."
  fi

  if [ "$failed" -ne 0 ]; then
    echo "Deployment configuration is incomplete. Nothing was built, stopped or replaced."
    return 1
  fi

  echo "Deployment configuration validated (database provider: ${DATABASE_PROVIDER})."
  return 0
}

# ---------------------------------------------------------------------------
# Database backup, taken BEFORE anything is stopped, built or replaced.
#
# A failed backup fails the deploy. The startup reconciler is additive and never
# drops a table holding rows, but "additive" is a property of the code, not a
# guarantee about the disk — and the first deploy after a schema change is
# exactly when a restore point matters. See deploy/README.md.
# ---------------------------------------------------------------------------
BACKUP_DIR="${BACKUP_DIR:-/opt/job-tracker/backups}"

backup_database() {
  if [ "${DEPLOY_SKIP_DB_BACKUP:-false}" = "true" ]; then
    echo "WARNING: DEPLOY_SKIP_DB_BACKUP=true — deploying with no restore point. Emergency use only."
    return 0
  fi

  # validate_deploy_config has already established this is set and recognised.
  local provider
  provider="$(printf '%s' "$DATABASE_PROVIDER" | tr '[:upper:]' '[:lower:]')"

  local stamp
  stamp="$(date -u +%Y%m%dT%H%M%SZ)"
  mkdir -p "$BACKUP_DIR"

  if [ "$provider" = "sqlite" ]; then
    # SQLite lives in the jobtracker_data volume. Tar it from a throwaway container
    # so the host needs no sqlite tooling and no knowledge of the volume layout.
    #
    # docker compose prefixes volume names with the project name, which defaults to
    # the directory name. Naming the bare volume here would silently CREATE an empty
    # one and back that up instead, so resolve it and fail if it is not there.
    local volume="${COMPOSE_PROJECT_NAME:-$(basename "$PWD")}_jobtracker_data"
    if ! docker volume inspect "$volume" >/dev/null 2>&1; then
      echo "SQLite data volume '${volume}' does not exist. Aborting deploy."
      echo "    Volumes are prefixed with the compose project name; set COMPOSE_PROJECT_NAME if it differs."
      echo "    Available: $(docker volume ls -q | grep jobtracker_data | tr '\n' ' ')"
      return 1
    fi

    local target="$BACKUP_DIR/jobtracker-sqlite-${stamp}.tar.gz"
    echo "Backing up SQLite data volume '${volume}' to ${target}"
    # The archive path is built inside the container: passing /backup/... as an argument
    # gets rewritten by MSYS path translation when the script is run from Git Bash.
    if ! docker run --rm \
        -v "$volume":/data:ro \
        -v "$BACKUP_DIR":/backup \
        -e ARCHIVE_NAME="$(basename "$target")" \
        alpine:3 sh -c 'tar czf "/backup/$ARCHIVE_NAME" -C /data .'; then
      echo "SQLite volume backup FAILED. Aborting deploy."
      rm -f "$target"
      return 1
    fi
    verify_volume_backup "$target"
    return $?
  fi

  local cs="${JOBTRACKER_CONNECTION_STRING:-}"
  if [ -z "$cs" ]; then
    echo "DATABASE_PROVIDER=${provider} but JOBTRACKER_CONNECTION_STRING is empty. Aborting deploy."
    return 1
  fi

  # Parse the ADO.NET connection string. Keys are case-insensitive in .NET, so match that.
  local db_host db_port db_name db_user db_pass
  db_host="$(sed -n 's/.*[Ss]erver=\([^;]*\).*/\1/p' <<<"$cs")"
  db_port="$(sed -n 's/.*[Pp]ort=\([^;]*\).*/\1/p' <<<"$cs")"
  db_name="$(sed -n 's/.*[Dd]atabase=\([^;]*\).*/\1/p' <<<"$cs")"
  db_user="$(sed -n 's/.*[Uu]ser[ ]*[Ii]*[Dd]*=\([^;]*\).*/\1/p' <<<"$cs")"
  db_pass="$(sed -n 's/.*[Pp]assword=\([^;]*\).*/\1/p' <<<"$cs")"
  db_port="${db_port:-3306}"

  if [ -z "$db_host" ] || [ -z "$db_name" ] || [ -z "$db_user" ]; then
    echo "Could not parse host/database/user from JOBTRACKER_CONNECTION_STRING. Aborting deploy."
    return 1
  fi

  local target="$BACKUP_DIR/jobtracker-${db_name}-${stamp}.sql.gz"
  echo "Backing up MariaDB database '${db_name}' on ${db_host}:${db_port} to ${target}"

  # The password goes via MYSQL_PWD, never on the command line, so it cannot leak
  # into the process list or the deploy log.
  local dump_status=0
  if command -v mariadb-dump >/dev/null 2>&1; then
    MYSQL_PWD="$db_pass" mariadb-dump \
      --host="$db_host" --port="$db_port" --user="$db_user" \
      --single-transaction --routines --events --quick \
      "$db_name" 2>/tmp/jobtracker-backup.err | gzip -c > "$target" || dump_status=$?
  elif command -v mysqldump >/dev/null 2>&1; then
    MYSQL_PWD="$db_pass" mysqldump \
      --host="$db_host" --port="$db_port" --user="$db_user" \
      --single-transaction --routines --events --quick \
      "$db_name" 2>/tmp/jobtracker-backup.err | gzip -c > "$target" || dump_status=$?
  else
    # No client on the host: run one in a container. --network host so the same
    # host/port from the connection string resolves identically.
    docker run --rm --network host -e MYSQL_PWD="$db_pass" mariadb:11 \
      mariadb-dump --host="$db_host" --port="$db_port" --user="$db_user" \
      --single-transaction --routines --events --quick \
      "$db_name" 2>/tmp/jobtracker-backup.err | gzip -c > "$target" || dump_status=$?
  fi

  if [ "$dump_status" -ne 0 ]; then
    echo "Database dump FAILED (exit ${dump_status}). Aborting deploy."
    sed -e 's/password=[^ ]*/password=***/gI' /tmp/jobtracker-backup.err >&2 || true
    rm -f "$target"
    return 1
  fi

  verify_sql_backup "$target"
}

# A backup that exists but is empty, truncated, or is the wrong KIND of backup is
# worse than none, because it looks like a restore point. Each provider gets the
# check that proves its own format — a tar cannot pass the dump check and a dump
# cannot pass the archive check.
#
# Note on `set +o pipefail` below: `grep -q` exits at the first match, which
# SIGPIPEs the decompressor upstream. Under pipefail that fails the pipeline and
# would reject a perfectly good backup.

verify_backup_file_shape() {
  local target="$1" kind="$2"

  if [ ! -s "$target" ]; then
    echo "Backup file ${target} is missing or empty. Aborting deploy."
    rm -f "$target"
    return 1
  fi

  if ! gzip -t "$target" 2>/dev/null; then
    echo "Backup ${target} is not a valid gzip archive (truncated ${kind}?). Aborting deploy."
    rm -f "$target"
    return 1
  fi

  return 0
}

report_backup() {
  local target="$1" size
  size="$(du -h "$target" | cut -f1)"
  echo "Backup verified: ${target} (${size})"
  echo "Retention is manual — old backups in ${BACKUP_DIR} are never deleted automatically."
  return 0
}

verify_sql_backup() {
  local target="$1"

  verify_backup_file_shape "$target" "dump" || return 1

  local has_schema=1 has_trailer=1
  set +o pipefail
  if gzip -dc "$target" | grep -q "CREATE TABLE"; then has_schema=0; fi
  # mariadb-dump / mysqldump write "-- Dump completed on ..." as the last line.
  # Its absence means the dump died partway through.
  if gzip -dc "$target" | grep -q "Dump completed"; then has_trailer=0; fi
  set -o pipefail

  if [ "$has_schema" -ne 0 ]; then
    echo "Backup ${target} contains no CREATE TABLE statement — it is not a usable dump. Aborting deploy."
    rm -f "$target"
    return 1
  fi

  if [ "$has_trailer" -ne 0 ]; then
    echo "Backup ${target} has no dump trailer — it is truncated. Aborting deploy."
    rm -f "$target"
    return 1
  fi

  report_backup "$target"
}

verify_volume_backup() {
  local target="$1"

  verify_backup_file_shape "$target" "archive" || return 1

  # Prove this is the DATABASE volume, not merely a non-empty tar. This is what
  # catches an archive taken from the wrong volume, or from a freshly created
  # empty one.
  # Listed in the same container that wrote it, not with the host's tar: GNU tar
  # reads a leading "C:/" as an rsh host and fails outright when this script runs
  # from Git Bash. Docker is already required to have produced the archive.
  local has_db=1
  if docker run --rm \
      -v "$(dirname "$target")":/backup:ro \
      -e ARCHIVE_NAME="$(basename "$target")" \
      alpine:3 sh -c 'tar tzf "/backup/$ARCHIVE_NAME" | grep -q "jobtracker\.db"' >/dev/null 2>&1; then
    has_db=0
  fi

  if [ "$has_db" -ne 0 ]; then
    echo "Archive ${target} contains no jobtracker.db — it is not a SQLite data backup. Aborting deploy."
    rm -f "$target"
    return 1
  fi

  report_backup "$target"
}

if ! validate_deploy_config; then
  exit 1
fi

if ! backup_database; then
  exit 1
fi

build_core_with_recovery() {
  if compose build backend frontend; then
    return 0
  fi

  echo "docker compose build for core services failed. Attempting one cleanup + retry because layer extraction can fail on constrained hosts."
  docker builder prune -af >/dev/null 2>&1 || true
  docker system prune -f >/dev/null 2>&1 || true
  compose build --no-cache backend frontend
}

build_ai_with_recovery() {
  if compose build ai-service; then
    return 0
  fi

  echo "docker compose build for ai-service failed. Attempting one cleanup + retry because layer extraction can fail on constrained hosts."
  docker image rm -f app-ai-service:latest 2>/dev/null || true
  docker builder prune -af >/dev/null 2>&1 || true
  docker system prune -f >/dev/null 2>&1 || true
  compose build --no-cache ai-service
}

compose pull || true
build_core_with_recovery
if [ "$DEPLOY_BUILD_AI_SERVICE" = "true" ]; then
  build_ai_with_recovery
else
  echo "Skipping ai-service rebuild during deploy (set DEPLOY_BUILD_AI_SERVICE=true to rebuild it)."
fi
# Force recreation so updated port mappings, env vars, and container config always apply on deploy.
compose up -d --force-recreate --remove-orphans backend frontend
if [ "$DEPLOY_BUILD_AI_SERVICE" = "true" ]; then
  # Ollama is opt-in (compose "bundled-ollama" profile). Deploys reuse an
  # existing/shared Ollama via OLLAMA_BASE_URL instead of starting a duplicate.
  compose up -d --force-recreate ai-service
fi

if [ -n "${OLLAMA_MODEL:-}" ]; then
  echo "Post-deploy Ollama warmup enabled for model: ${OLLAMA_MODEL}"
  ./scripts/start-ollama-cv.sh
fi

sleep 5
compose ps

backend_status="$(compose ps backend --format '{{.State}}' 2>/dev/null | head -n 1 | tr '[:upper:]' '[:lower:]')"
if [ "$backend_status" != "running" ]; then
  echo "Backend service is not healthy after deploy (state: ${backend_status:-unknown})."
  compose logs --tail=200 backend || true
  exit 1
fi

ai_status="$(compose ps ai-service --format '{{.State}}' 2>/dev/null | head -n 1 | tr '[:upper:]' '[:lower:]')"
if [ "$ai_status" != "running" ]; then
  echo "AI service is not healthy after deploy (state: ${ai_status:-unknown}). Continuing because AI is not a deploy gate for the core app."
  compose logs --tail=200 ai-service || true
fi

if [ -n "${APP_PUBLIC_BASE_URL:-}" ]; then
  public_base="${APP_PUBLIC_BASE_URL%/}"
  auth_config_body_file="$(mktemp)"
  auth_config_headers_file="$(mktemp)"
  cleanup_public_check() {
    rm -f "$auth_config_body_file" "$auth_config_headers_file"
  }
  trap cleanup_public_check EXIT

  echo "Running public smoke check against ${public_base}"
  if ! curl -fsS "${public_base}/" >/dev/null; then
    echo "Public frontend check failed for ${public_base}/"
    exit 1
  fi

  if ! curl -fsS -D "$auth_config_headers_file" -o "$auth_config_body_file" "${public_base}/api/auth/config"; then
    echo "Public API smoke check failed for ${public_base}/api/auth/config"
    exit 1
  fi

  content_type="$(awk 'BEGIN{IGNORECASE=1} /^content-type:/ {print $2}' "$auth_config_headers_file" | tr -d '\r' | tail -n 1)"
  if [[ "$content_type" != application/json* ]]; then
    echo "Public API smoke check returned unexpected content type: ${content_type:-missing}"
    echo "First bytes of response:"
    head -c 200 "$auth_config_body_file" || true
    exit 1
  fi

  if ! grep -q 'requireAuth' "$auth_config_body_file"; then
    echo "Public API smoke check returned JSON without requireAuth."
    cat "$auth_config_body_file"
    exit 1
  fi

  trap - EXIT
  cleanup_public_check
fi

# Clean up old legacy container name if it still exists from pre-rename deployments.
docker rm -f app-summarizer-1 2>/dev/null || true

echo "Deployment complete: ${APP_VERSION} ${APP_COMMIT_SHA}"