fix(release): verify core health after automatic image rollback
This commit is contained in:
+10
-2
@@ -660,10 +660,18 @@ finish_deploy() {
|
||||
|
||||
if [ "$status" -ne 0 ] && [ "$replacement_started" = true ]; then
|
||||
if [ -n "$previous_backend_ref" ] && [ -n "$previous_frontend_ref" ]; then
|
||||
echo "Deployment failed after core replacement; restoring previous healthy images."
|
||||
echo "Deployment failed after core replacement; restarting previous core images."
|
||||
if BACKEND_IMAGE="$previous_backend_ref" FRONTEND_IMAGE="$previous_frontend_ref" \
|
||||
compose up -d --force-recreate --remove-orphans backend frontend; then
|
||||
echo "Previous core release restored. Deployment remains failed; inspect the logs above."
|
||||
if curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/" >/dev/null &&
|
||||
curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/health" >/dev/null &&
|
||||
curl -fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30 "${APP_PUBLIC_BASE_URL%/}/ready" >/dev/null; then
|
||||
echo "Previous core images restarted and public core checks passed. Deployment remains failed; inspect the logs above."
|
||||
else
|
||||
echo "Previous core images restarted, but rollback health verification failed. Manual recovery is required."
|
||||
compose ps || true
|
||||
compose logs --tail=200 backend frontend || true
|
||||
fi
|
||||
else
|
||||
echo "Automatic core rollback failed. Manual recovery is required."
|
||||
compose ps || true
|
||||
|
||||
@@ -10,12 +10,14 @@ BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which(
|
||||
|
||||
|
||||
class DeployRollbackTests(unittest.TestCase):
|
||||
def exit_deploy(self, status, *, replaced=True, backend=True, frontend=True, compose_status=0):
|
||||
def exit_deploy(self, status, *, replaced=True, backend=True, frontend=True, compose_status=0, failed_probe='', diagnostics_status=0):
|
||||
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
|
||||
start = source.index('finish_deploy() {')
|
||||
handler = source[start:source.index('\ntrap finish_deploy EXIT\n', start) + len('\ntrap finish_deploy EXIT\n')]
|
||||
script = '\n'.join([
|
||||
'set -euo pipefail',
|
||||
'exec 3>&1',
|
||||
'APP_PUBLIC_BASE_URL=https://synthetic.invalid/',
|
||||
f'replacement_started={str(replaced).lower()}',
|
||||
'previous_backend_ref=' + ('synthetic/backend:previous' if backend else ''),
|
||||
'previous_frontend_ref=' + ('synthetic/frontend:previous' if frontend else ''),
|
||||
@@ -23,11 +25,15 @@ class DeployRollbackTests(unittest.TestCase):
|
||||
'auth_config_headers_file=synthetic-headers',
|
||||
'rm() { printf "CALL rm %s\\n" "$*"; }',
|
||||
'docker() { echo "CALL unexpected docker"; return 99; }',
|
||||
'curl() { echo "CALL unexpected curl"; return 99; }',
|
||||
'curl() {',
|
||||
' printf "CALL curl %s\\n" "$*" >&3;',
|
||||
f' if [ "${{@: -1}}" = "https://synthetic.invalid{failed_probe}" ]; then return 22; fi;',
|
||||
' return 0;',
|
||||
'}',
|
||||
'compose() {',
|
||||
' printf "CALL compose %s [%s] [%s]\\n" "$*" "${BACKEND_IMAGE:-}" "${FRONTEND_IMAGE:-}";',
|
||||
f' if [ "$1" = up ]; then return {compose_status}; fi;',
|
||||
' return 0;',
|
||||
f' return {diagnostics_status};',
|
||||
'}',
|
||||
handler,
|
||||
f'exit {status}',
|
||||
@@ -49,11 +55,28 @@ class DeployRollbackTests(unittest.TestCase):
|
||||
self.assertEqual([
|
||||
'CALL compose up -d --force-recreate --remove-orphans backend frontend '
|
||||
'[synthetic/backend:previous] [synthetic/frontend:previous]',
|
||||
*self.probes(),
|
||||
], calls)
|
||||
self.assertIn('public core checks passed', stdout)
|
||||
self.assertIn('Deployment remains failed', stdout)
|
||||
|
||||
@staticmethod
|
||||
def probes():
|
||||
flags = '-fsS --connect-timeout 5 --max-time 5 --retry 5 --retry-delay 2 --retry-connrefused --retry-max-time 30'
|
||||
return [f'CALL curl {flags} https://synthetic.invalid{path}' for path in ['/', '/health', '/ready']]
|
||||
|
||||
def test_each_failed_health_probe_preserves_failure_and_requires_manual_recovery(self):
|
||||
for index, path in enumerate(['/', '/health', '/ready']):
|
||||
with self.subTest(path=path):
|
||||
stdout, calls = self.exit_deploy(41, failed_probe=path, diagnostics_status=17)
|
||||
self.assertEqual(self.probes()[:index + 1], calls[1:-2])
|
||||
self.assertEqual(['CALL compose ps [] []', 'CALL compose logs --tail=200 backend frontend [] []'], calls[-2:])
|
||||
self.assertIn('rollback health verification failed', stdout)
|
||||
self.assertIn('Manual recovery is required', stdout)
|
||||
self.assertNotIn('public core checks passed', stdout)
|
||||
|
||||
def test_rollback_failure_retains_original_status_and_collects_diagnostics(self):
|
||||
stdout, calls = self.exit_deploy(41, compose_status=19)
|
||||
stdout, calls = self.exit_deploy(41, compose_status=19, diagnostics_status=17)
|
||||
self.assertEqual([
|
||||
'CALL compose up -d --force-recreate --remove-orphans backend frontend '
|
||||
'[synthetic/backend:previous] [synthetic/frontend:previous]',
|
||||
|
||||
Reference in New Issue
Block a user