fix(release): verify core health after automatic image rollback

This commit is contained in:
cesnimda
2026-11-11 07:19:45 +01:00
parent 24a629972e
commit a39ac84729
2 changed files with 37 additions and 6 deletions
+10 -2
View File
@@ -660,10 +660,18 @@ finish_deploy() {
if [ "$status" -ne 0 ] && [ "$replacement_started" = true ]; then
if [ -n "$previous_backend_ref" ] && [ -n "$previous_frontend_ref" ]; then
echo "Deployment failed after core replacement; restoring previous healthy images."
echo "Deployment failed after core replacement; restarting previous core images."
if BACKEND_IMAGE="$previous_backend_ref" FRONTEND_IMAGE="$previous_frontend_ref" \
compose up -d --force-recreate --remove-orphans backend frontend; then
echo "Previous core release restored. Deployment remains failed; inspect the logs above."
if curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/" >/dev/null &&
curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/health" >/dev/null &&
curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/ready" >/dev/null; then
echo "Previous core images restarted and public core checks passed. Deployment remains failed; inspect the logs above."
else
echo "Previous core images restarted, but rollback health verification failed. Manual recovery is required."
compose ps || true
compose logs --tail=200 backend frontend || true
fi
else
echo "Automatic core rollback failed. Manual recovery is required."
compose ps || true
+27 -4
View File
@@ -10,12 +10,14 @@ BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which(
class DeployRollbackTests(unittest.TestCase):
def exit_deploy(self, status, *, replaced=True, backend=True, frontend=True, compose_status=0):
def exit_deploy(self, status, *, replaced=True, backend=True, frontend=True, compose_status=0, failed_probe='', diagnostics_status=0):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
start = source.index('finish_deploy() {')
handler = source[start:source.index('\ntrap finish_deploy EXIT\n', start) + len('\ntrap finish_deploy EXIT\n')]
script = '\n'.join([
'set -euo pipefail',
'exec 3>&1',
'APP_PUBLIC_BASE_URL=https://synthetic.invalid/',
f'replacement_started={str(replaced).lower()}',
'previous_backend_ref=' + ('synthetic/backend:previous' if backend else ''),
'previous_frontend_ref=' + ('synthetic/frontend:previous' if frontend else ''),
@@ -23,11 +25,15 @@ class DeployRollbackTests(unittest.TestCase):
'auth_config_headers_file=synthetic-headers',
'rm() { printf "CALL rm %s\\n" "$*"; }',
'docker() { echo "CALL unexpected docker"; return 99; }',
'curl() { echo "CALL unexpected curl"; return 99; }',
'curl() {',
' printf "CALL curl %s\\n" "$*" >&3;',
f' if [ "${{@: -1}}" = "https://synthetic.invalid{failed_probe}" ]; then return 22; fi;',
' return 0;',
'}',
'compose() {',
' printf "CALL compose %s [%s] [%s]\\n" "$*" "${BACKEND_IMAGE:-}" "${FRONTEND_IMAGE:-}";',
f' if [ "$1" = up ]; then return {compose_status}; fi;',
' return 0;',
f' return {diagnostics_status};',
'}',
handler,
f'exit {status}',
@@ -49,11 +55,28 @@ class DeployRollbackTests(unittest.TestCase):
self.assertEqual([
'CALL compose up -d --force-recreate --remove-orphans backend frontend '
'[synthetic/backend:previous] [synthetic/frontend:previous]',
*self.probes(),
], calls)
self.assertIn('public core checks passed', stdout)
self.assertIn('Deployment remains failed', stdout)
@staticmethod
def probes():
flags = '-fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30'
return [f'CALL curl {flags} https://synthetic.invalid{path}' for path in ['/', '/health', '/ready']]
def test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery(self):
for index, path in enumerate(['/', '/health', '/ready']):
with self.subTest(path=path):
stdout, calls = self.exit_deploy(41, failed_probe=path, diagnostics_status=17)
self.assertEqual(self.probes()[:index + 1], calls[1:-2])
self.assertEqual(['CALL compose ps [] []', 'CALL compose logs --tail=200 backend frontend [] []'], calls[-2:])
self.assertIn('rollback health verification failed', stdout)
self.assertIn('Manual recovery is required', stdout)
self.assertNotIn('public core checks passed', stdout)
def test_rollback_failure_retains_original_status_and_collects_diagnostics(self):
stdout, calls = self.exit_deploy(41, compose_status=19)
stdout, calls = self.exit_deploy(41, compose_status=19, diagnostics_status=17)
self.assertEqual([
'CALL compose up -d --force-recreate --remove-orphans backend frontend '
'[synthetic/backend:previous] [synthetic/frontend:previous]',