Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f19e7ebf4a | |||
| 653f8d04d4 | |||
| edd2aa9ae4 |
@@ -3,3 +3,7 @@
|
|||||||
The deployment script and Gitea workflow live in this directory and `.gitea/workflows/`. The current deployment, health/readiness, rollback, and secret-handling guidance is in the repository [infrastructure guide](../docs/infrastructure/README.md), especially [deployment](../docs/infrastructure/deployment.md) and [disaster recovery](../docs/infrastructure/disaster-recovery.md).
|
The deployment script and Gitea workflow live in this directory and `.gitea/workflows/`. The current deployment, health/readiness, rollback, and secret-handling guidance is in the repository [infrastructure guide](../docs/infrastructure/README.md), especially [deployment](../docs/infrastructure/deployment.md) and [disaster recovery](../docs/infrastructure/disaster-recovery.md).
|
||||||
|
|
||||||
Keep the server `.env` outside the repository. Do not run production deployment, restore, migration, or credential rotation as part of local documentation work.
|
Keep the server `.env` outside the repository. Do not run production deployment, restore, migration, or credential rotation as part of local documentation work.
|
||||||
|
|
||||||
|
MariaDB backup connection parsing uses `scripts/parse-backup-connection.py` (Python 3, already required by deployment preflight). Keys are case-insensitive and support the documented host/user/password/database aliases. Single/double-quoted values may contain semicolons and doubled quote characters. The dump contract requires one TCP host, a database and user, with port 1-65535 (default 3306); an empty password is retained. Duplicate aliases, control characters, host lists/socket paths and database path separators are rejected before dumping. This deliberately bounded parser is not the full MySqlConnector configuration engine. Additional connection options do not change existing dump-client TLS behavior; that qualification remains separate.
|
||||||
|
|
||||||
|
Run `python scripts/test-deploy-backup.py` for the offline synthetic dump-boundary regression. It extracts only the backup function and never executes the deployment entry point, contacts a database or reads the production environment.
|
||||||
|
|||||||
+11
-11
@@ -326,19 +326,19 @@ backup_database() {
|
|||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Parse the ADO.NET connection string. Keys are case-insensitive in .NET, so match that.
|
# Keep secrets off argv; parse aliases/quoted separators without evaluating shell text.
|
||||||
local db_host db_port db_name db_user db_pass
|
local db_host db_port db_name db_user db_pass parsed_connection
|
||||||
db_host="$(sed -n 's/.*[Ss]erver=\([^;]*\).*/\1/p' <<<"$cs")"
|
local -a connection_fields=()
|
||||||
db_port="$(sed -n 's/.*[Pp]ort=\([^;]*\).*/\1/p' <<<"$cs")"
|
if ! parsed_connection="$(printf '%s' "$cs" | python3 scripts/parse-backup-connection.py)"; then
|
||||||
db_name="$(sed -n 's/.*[Dd]atabase=\([^;]*\).*/\1/p' <<<"$cs")"
|
echo "Could not parse JOBTRACKER_CONNECTION_STRING for backup. Aborting deploy."
|
||||||
db_user="$(sed -n 's/.*[Uu]ser[ ]*[Ii]*[Dd]*=\([^;]*\).*/\1/p' <<<"$cs")"
|
|
||||||
db_pass="$(sed -n 's/.*[Pp]assword=\([^;]*\).*/\1/p' <<<"$cs")"
|
|
||||||
db_port="${db_port:-3306}"
|
|
||||||
|
|
||||||
if [ -z "$db_host" ] || [ -z "$db_name" ] || [ -z "$db_user" ]; then
|
|
||||||
echo "Could not parse host/database/user from JOBTRACKER_CONNECTION_STRING. Aborting deploy."
|
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
mapfile -t connection_fields <<< "$parsed_connection"
|
||||||
|
db_host="${connection_fields[0]}"
|
||||||
|
db_port="${connection_fields[1]}"
|
||||||
|
db_name="${connection_fields[2]}"
|
||||||
|
db_user="${connection_fields[3]}"
|
||||||
|
db_pass="${connection_fields[4]}"
|
||||||
|
|
||||||
local target="$BACKUP_DIR/jobtracker-${db_name}-${BACKUP_STAMP}.sql.gz"
|
local target="$BACKUP_DIR/jobtracker-${db_name}-${BACKUP_STAMP}.sql.gz"
|
||||||
echo "Backing up MariaDB database '${db_name}' on ${db_host}:${db_port} to ${target}"
|
echo "Backing up MariaDB database '${db_name}' on ${db_host}:${db_port} to ${target}"
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# Preliminary release discovery
|
||||||
|
|
||||||
|
2026-11-02. Branch `planning/release-readiness-discovery`; accepted conditional development baseline `5cb903b989742b7a96cb22ddf4c0e24993b18980`. Stage 1 IN_PROGRESS. No release candidate nominated. Production NO-GO. This report records source inspection and a bounded offline check, not full release qualification.
|
||||||
|
|
||||||
|
| Component / inspected owner | Dependency / failure boundary | Required release evidence |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `job-tracker-ui/package.json`, `Dockerfile`, `nginx.conf.template` | Next static export, React/MUI and React Router; nginx serves `out`, proxies `/api/`, `/health`, `/ready`; configured host and forwarded-header handling | Exact build/image, browser journeys and deployed proxy/TLS identity |
|
||||||
|
| `JobTrackerApi/Program.cs`, `JobTrackerApi.csproj` | net9.0, EF 9.0.14, Pomelo 9.0.0; authentication/authorization then rate limiting/session middleware and controllers | Endpoint-by-endpoint ownership/session/CSRF/error tests; source registration alone is not security proof |
|
||||||
|
| `JobTrackerApi/Data/DatabaseConfiguration.cs` | Defaults to MariaDB; missing connection throws; SQLite requires explicit transitional selection | Actual MariaDB migration/upgrade/rollback and preservation evidence; retain legacy paths |
|
||||||
|
| `docker-compose.database.yml` | Development persistent volume and separate integration tmpfs service; MariaDB 12.2.2 pinned digest; loopback ports | Isolated provider tests; no claim of equality with production |
|
||||||
|
| Career/CV domain and PDF | Program registers CareerProfileService, ApplicationAssetsService, CV/letter renderers and Playwright exporter | Canonical facts, null-as-inherit, revisions, historical artifacts and physical print remain separate contracts |
|
||||||
|
| Background services | Program registers backup, rules, reminders, saved search, export, enrichment, summarizer probe, AI operation, retention, email recovery and account deletion workers | Verify configuration, failure recovery and tenant isolation for enabled workers; registration is not runtime activation |
|
||||||
|
| Auth/mail/provider integrations | Architecture documents Google/Microsoft identity, Gmail/Graph/IMAP/SMTP, NAV, optional translation, local/external AI, Stripe | Synthetic local tests separate from licensed/provider/account delivery and consent |
|
||||||
|
| `.gitea/workflows/ci-deploy.yml` | Test pipeline; deployment limited by source condition to push on `refs/heads/main` | Fresh hosted artifact/commit/runner evidence; development branch push is not hosted qualification |
|
||||||
|
| `.gitea/workflows/mariadb-integration.yml` | Isolated runner label, explicit initial exit-1 isolation gate | Operator isolation evidence and authorised workflow review before enabling; do not remove gate during discovery |
|
||||||
|
| Existing backup/deployment runbooks | Database, attachments/CV artifacts, keys and deletion tombstones; code rollback differs from data restore | Disposable restore with independent content/decryption/deletion checks; approved RPO/RTO and failure rehearsal |
|
||||||
|
|
||||||
|
Public boundary currently inspected: nginx host routing, `/health`, `/ready`, `/api/`; API controllers are mapped but a complete anonymous/authenticated endpoint census is still PENDING. External mail/job/AI requests, uploaded documents, browser credentials, database records and stored artifacts are separate trust boundaries. No live endpoints, Docker engines, databases, credentials, models or production services were accessed.
|
||||||
|
|
||||||
|
Local development, test and production are not interchangeable: integration uses a generated project and disposable service; production needs the external network, server-side environment, actual ingress and operator controls. Current architecture/runbooks describe these differences; exact intended staging/production configuration and image identity remain unverified.
|
||||||
|
|
||||||
|
The latest handoff's source-bound PDF metric/visual approvals supersede the older acceptance paragraph describing unchanged S01 coordinate failures. They are preserved for the approved development scope. Physical print, device, AT and language review remain open. No export was rerun because no renderer/font/fixture/application source changed.
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# Discovery baseline and actual checks
|
||||||
|
|
||||||
|
Date 2026-11-02. Source `5cb903b989742b7a96cb22ddf4c0e24993b18980`; application freeze `ba04d817fed26d659bfa2a56b089d6cac851dc34`. No accepted post-roadmap release candidate, build identifier or release image exists in this discovery checkpoint.
|
||||||
|
|
||||||
|
Fresh check: `rtk proxy python scripts/test-mariadb-integration.py` exited 0; **5 tests passed** in 0.030 seconds. This existing offline suite mocks subprocess execution. The printed cleanup failure/exit 17 and setup error/exit 1 are intentional negative-test outcomes asserted by the suite, not a real container failure. No MariaDB/Docker execution occurred. This proves five runner guards, not database behavior, hosted isolation or a full regression.
|
||||||
|
|
||||||
|
Read-only source checks confirmed the MariaDB default/missing-connection fail-closed behavior and the hosted workflow's explicit isolation blocker. Refer to SYSTEM_INVENTORY for inspected paths. `git diff --check` is a documentation hygiene check, not product qualification.
|
||||||
|
|
||||||
|
Historical test totals, PDF exports, security scans and provider runs remain attributed to their original evidence. They are not fresh runs here. `docs/automation/handoffs/pre-phase10/FINAL_DEVELOPMENT_ENTRY.md` records the latest approved unchanged-source applicability and narrow PDF metric/visual closure. Direct chat does not adopt or activate its historical autonomous-controller instructions.
|
||||||
|
|
||||||
|
NOT RUN: full backend/frontend/browser/Python/provider suites, builds, container validation, security scan/advisory refresh, performance, native review, deployment smoke, restore or rollback rehearsal. These require the actual release-stage source/environment and their own applicable gates. Discovery does not mark any mandatory release gate PASS merely because historical tests exist.
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# Recovery implementation review and rehearsal plan
|
||||||
|
|
||||||
|
2026-11-02; source `5cb903b989742b7a96cb22ddf4c0e24993b18980`. Read-only discovery. The entire 765-line `deploy/deploy.sh` was inspected, never executed or sourced. No operational keys, credentials, environment files, databases, containers or remote storage were accessed.
|
||||||
|
|
||||||
|
## Distinct recovery mechanisms
|
||||||
|
|
||||||
|
| Owner | Verified source behavior | Qualification boundary |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `SqliteDatabaseBackupRunner` | SQLite-only VACUUM snapshot and count retention; explicitly unsupported for MariaDB | Does not prove MariaDB backups exist |
|
||||||
|
| `BackupController` | Authenticated, encrypted user-data export using application Data Protection; includes structured records and attachment metadata | Not a server-wide database/files/keys recovery set; do not use it as disaster recovery proof |
|
||||||
|
| `deploy/deploy.sh:backup_database` | Requires explicit provider; SQL dump for MariaDB, volume archive for SQLite; rejects missing/empty/malformed/trailerless dumps | Format checks do not restore or compare records; emergency skip exists and is not approved here |
|
||||||
|
| `backup_recovery_volumes` | MariaDB additionally archives required application data volume; independently archives tombstones when present | Consistency between dump/files/tombstones and current key availability still needs isolated rehearsal |
|
||||||
|
| Retention/off-host copy | Daily/monthly retention and optional rclone copy | Source message calls remote encrypted but source does not establish actual remote encryption configuration; off-host configuration and recovery remain unverified |
|
||||||
|
| `finish_deploy` | On failure after replacement, restarts previous backend/frontend image references when both exist; retains failed exit | No database reversal, no previous AI image restoration, and no post-rollback health check in this handler |
|
||||||
|
| Tombstone store and replay | Validates ledger records; Program invokes `StageRestoredAccountsAsync` at startup; deleted identities are staged again | Test restored data with latest independent tombstones; corrupt ledger must fail closed. Existing AccountDeletionTests include restore replay, not a complete operational restore |
|
||||||
|
|
||||||
|
Build retry paths prune Docker build/system cache and deployment may warm an Ollama model. Therefore even a proposed "local deploy smoke" must not run this script against the shared workstation. No such action was taken.
|
||||||
|
|
||||||
|
## RR-11: backup connection parser mismatch - reproduced
|
||||||
|
|
||||||
|
The script's host `sed` expression accepts `Server=` but returns no host for `SERVER=` and `Host=`. The latter is a documented MySqlConnector alias ([official connection options](https://mysqlconnector.net/connection-options/), checked 2026-11-02). The script then fails its required-host check before dump execution. Impact: a valid application connection spelling can prevent deployment backup/deployment; no observed data loss or actual production impact. Engineering priority proposed P2; not a security vulnerability rating.
|
||||||
|
|
||||||
|
Three synthetic cases used only loopback and dummy identifiers. Exact source expression was extracted from the `db_host` assignment, then executed through Git Bash with `echo <shell-quoted synthetic input> | sed -n <shell-quoted source expression>`. Results: canonical `Server` -> `127.0.0.1`; uppercase `SERVER` -> empty; alias `Host` -> empty. Assertions confirmed all three results. No connection was opened. Initial direct Windows-to-sed invocations failed the positive-control assertion and were discarded as invalid harness evidence; the shell-native rerun passed its positive control.
|
||||||
|
|
||||||
|
Proposed future remediation: parse with the same supported connection-string semantics as the API, or explicitly validate/document a narrower deploy contract before any operational step. Cover aliases/case, quoted separators, duplicate keys, missing required values and secret-redaction behavior. Do not print real input. Any implementation belongs to an authorised release remediation scope, not P10 cleanup.
|
||||||
|
|
||||||
|
## Isolated rehearsal specification - NOT RUN
|
||||||
|
|
||||||
|
1. Bind accepted candidate, image digests, migration history and disposable host ownership. Use synthetic accounts and a separate temporary MariaDB/files/key/tombstone set; never copy production data here without separate authority.
|
||||||
|
2. Seed two owners with canonical profiles, CV overrides, letter history, submitted packages, attachments, encrypted provider placeholders and completed deletion tombstones. Record independent counts and artifact hashes, without recording plaintext secrets.
|
||||||
|
3. Produce the actual supported dump/files/tombstone recovery set. Demonstrate a failed dump or missing required volume prevents promotion. Verify off-host retrieval/decryption separately if that integration is approved.
|
||||||
|
4. Restore into a second isolated environment. Independently compare rows/ownership/revisions/artifact bytes; exercise application decryption and document reopening. Restore an older database with newer tombstones and prove erased accounts cannot regain access. Inject a malformed ledger and assert fail-closed startup.
|
||||||
|
5. Rehearse prior-image rollback after failed replacement, then after a representative additive migration and writes. Check `/health`, `/ready`, exact version and authenticated read/write preservation. Document when image rollback is insufficient and database restore would lose newer writes.
|
||||||
|
6. Measure recovery duration and data-loss window against owner-approved RTO/RPO. No target or measurement exists in this checkpoint. Preserve cleanup and evidence identities; no production execution or release approval implied.
|
||||||
|
|
||||||
|
Fresh checks: Git Bash `bash -n deploy/deploy.sh` PASSED (syntax only); `python scripts/test-mariadb-workflow.py` PASSED 4/4 (source quarantine guards only). These are separate from the earlier 5/5 offline runner tests. Restore, failure injection, hosted isolation, live encryption and rollback health remain NOT TESTED.
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Preliminary release blocker reconciliation
|
||||||
|
|
||||||
|
Discovery only, 2026-11-02. These are qualification gaps unless a concrete defect is stated. No inferred P0/P1 vulnerability ratings or owner risk acceptance.
|
||||||
|
|
||||||
|
| ID | Historical item / current finding | Status / closure evidence needed |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| RR-01 | Phase 9/10 and production NO-GO | Phase 10 assessment now independently published, implementation still awaits exact approval. Later roadmap acceptance absent. BLOCKED for final RC |
|
||||||
|
| RR-02 | Old Linux 11/24 and later S01 failure narratives | Latest handoff records approved S01 36/36 per platform and owner visual acceptance for unchanged development output. Do not reopen these metrics without affected change; physical print/device remain NOT TESTED |
|
||||||
|
| RR-03 | Python advisory/native parser/image qualification | Latest scoped conditional disposition retained; current advisories and production-equivalent native isolation NOT TESTED. No new scan or risk waiver |
|
||||||
|
| RR-04 | Hosted MariaDB/isolation | Source explicitly fails before execution until isolated runner proof. BLOCKED; fresh local 5 guard tests do not close it |
|
||||||
|
| RR-05 | Human/device/provider | Actual AT/contrast, physical devices/print, native language and live provider accounts not qualified here. BLOCKED/NOT TESTED by lane |
|
||||||
|
| RR-06 | Old known-issues F19 SQLite fallback statement | Current DatabaseConfiguration defaults to MariaDB and throws without connection. Old wording is superseded for this source. SQLite retirement still BLOCKED on transfer/restore/parity gates |
|
||||||
|
| RR-07 | Old F01 stale-write statement | MariaDB acceptance documents bounded RD21 resolution; later handoff carries it. Do not reopen all old F01 findings based on stale index wording; final affected API census/regression remains required |
|
||||||
|
| RR-08 | MySQL TLS / dependency / production identity | Latest handoff retains MYSQL-TLS-01 and OPS-01. No live configuration read; current advisory disposition and deployed equivalence unverified |
|
||||||
|
| RR-09 | Backup/restore and rollback | Existing procedures and historical seeded rehearsal are not a current restore. Need isolated recovery-set completeness, decryption, tombstone behavior, post-write rollback compatibility and agreed RPO/RTO |
|
||||||
|
| RR-10 | AI quality/activation | Deterministic Docling fixtures are distinct from real-model semantic accuracy. Optional unqualified adapter remains disabled; no AI invocation authorised here |
|
||||||
|
| RR-11 | Backup host parsing rejects supported `Host=` alias; uppercase spelling also unrecognised | REPRODUCED with synthetic input and shell-native positive control. Deployment aborts before backup; proposed P2 compatibility defect. Fix scope awaits authorised release remediation |
|
||||||
|
| RR-12 | Rollback restores core images without post-rollback health qualification; no schema/data/AI reversal | VERIFIED source limitation, runtime impact NOT TESTED. Requires isolated post-migration/post-write recovery rehearsal; no rollback guarantee |
|
||||||
|
| RR-13 | Off-host encryption and recovery-set consistency not established by script success messages | NOT TESTED operational qualification. Verify remote encryption, retrieval/decryption, database/files/key/tombstone consistency in approved isolation |
|
||||||
|
|
||||||
|
Next remediation is evidence work: complete the source/endpoint/environment census and resolve historical records against their exact latest evidence. RR-11 now has a bounded future fix proposal in 11_BACKUP_RESTORE_ROLLBACK.md; no runtime implementation is authorised by this discovery checkpoint. Every fix needs minimal scope, targeted verification and the applicable stage approval.
|
||||||
@@ -0,0 +1,26 @@
|
|||||||
|
# Preliminary acceptance matrix
|
||||||
|
|
||||||
|
2026-11-02. **NO-GO for production.** This is a discovery snapshot, not Stage 7 final acceptance. No accepted final RC or production-equivalent build is nominated. PASS applies only to an executed gate, not historical source evidence.
|
||||||
|
|
||||||
|
| Mandatory gate | Current release verdict | Missing release evidence / dependency |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Functional regression | BLOCKED | Accepted final roadmap source; fresh applicable critical journeys |
|
||||||
|
| Frontend/UX | BLOCKED | Accepted design/implementation and final source-bound browser evidence |
|
||||||
|
| Accessibility | NOT TESTED | Actual AT/device/contrast/manual evidence plus automation |
|
||||||
|
| Authentication/authorization | NOT TESTED | Final endpoint/tenant/session qualification |
|
||||||
|
| Security assessment | BLOCKED | Dedicated security initiative start and predecessor gates; no scan run |
|
||||||
|
| Backend APIs | NOT TESTED | Final API census and applicable regression |
|
||||||
|
| MariaDB/migrations | BLOCKED | Hosted isolation and actual RC upgrade/recovery evidence; five offline guards are not this gate |
|
||||||
|
| CV/PDF fidelity | BLOCKED | Final RC source equivalence and retained scoped approvals; physical print still missing |
|
||||||
|
| AI integrations | BLOCKED | Explicit isolated-model qualification and semantic evidence |
|
||||||
|
| Email/OAuth | BLOCKED | Actual configured provider/consent/account verification |
|
||||||
|
| Performance/reliability | NOT TESTED | Approved load targets and bounded final-source measurements |
|
||||||
|
| Docker/infrastructure | NOT TESTED | Exact images/environment/configuration/ingress parity |
|
||||||
|
| CI/CD | BLOCKED | Hosted isolation and fresh actual run/artifacts |
|
||||||
|
| Backup/restore | NOT TESTED | Disposable complete recovery rehearsal with content assertions |
|
||||||
|
| Deployment/rollback | NOT TESTED | Authorised isolated rehearsal; production execution remains prohibited |
|
||||||
|
| Monitoring | NOT TESTED | Failure detection, delivery and redaction evidence |
|
||||||
|
| Documentation | NOT TESTED | Complete release-source/runbook reconciliation, beyond this partial discovery |
|
||||||
|
| Human/device/provider | BLOCKED | Real reviewers/devices/accounts and source-bound records |
|
||||||
|
|
||||||
|
No gate is marked FAIL without fresh evidence of unacceptable behavior. No missing qualification is marked PASS. Historical source-specific acceptance may be reused where permitted and unchanged, but cannot establish final release readiness by itself.
|
||||||
@@ -0,0 +1,11 @@
|
|||||||
|
# Release discovery handoff
|
||||||
|
|
||||||
|
Branch `planning/release-readiness-discovery`, baseline `5cb903b989742b7a96cb22ddf4c0e24993b18980`; documentation only. Stage 1 IN_PROGRESS, not complete. Full authoritative release prompt read. No final GO/NO-GO completion report or deployable release claimed.
|
||||||
|
|
||||||
|
Completed: initial source inventory, historical blocker reconciliation, preliminary 18-gate matrix and five fresh offline MariaDB runner guard tests. Preserved scoped PDF acceptance and separate model/provider/human/hosted/production gates.
|
||||||
|
|
||||||
|
Next authorised work: complete endpoint and environment census, map each failure boundary to existing test/runbook owners, and inspect remaining backup/restore/deployment implementation without executing it. Then prepare a concrete isolated-rehearsal plan. Final baseline/qualification depends on accepted roadmap source; do not run thousands of unchanged tests to fill this report.
|
||||||
|
|
||||||
|
Continuation: recovery implementation inspection and isolated rehearsal specification are now recorded in 11_BACKUP_RESTORE_ROLLBACK.md. RR-11 parser mismatch reproduced using three synthetic cases; RR-12/13 retain rollback/encryption qualification limits. Four source quarantine guard tests and shell syntax check passed. No deployment script execution. Next: endpoint/environment census, then remaining independent I18N/homepage audit work. Stage 1 remains IN_PROGRESS.
|
||||||
|
|
||||||
|
No production credentials/environment files were opened; no infrastructure, models, main branch or autonomous orchestrator changed. The earlier delegated researcher was interrupted by a service limit; parent supplied this partial checkpoint. No background agent remains active.
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# RR-11 bounded remediation
|
||||||
|
|
||||||
|
Authorised by the owner's Continuous Roadmap Execution instruction explicitly prioritising this fix. Branch `codex/release-backup-parser`, based on published discovery `653f8d04d469dcd52c883d4b98bea76b8adacada`. Separate from CLEAN-20, full release qualification and infrastructure operations.
|
||||||
|
|
||||||
|
Replaced five inconsistent sed expressions with one stdlib Python parser and a shell-safe field transfer. Case-insensitive aliases, quoted separators/doubled quotes, Unicode and empty passwords survive to the dump command. Invalid ports, incomplete/ambiguous/control-bearing values and unsupported multi-host/socket/path forms fail before dumping. No eval; connection input travels via stdin, not argv; errors never include input. No dependency added. Existing dump/TLS/retention/rollback behavior is otherwise unchanged.
|
||||||
|
|
||||||
|
## Evidence
|
||||||
|
|
||||||
|
- Regression first: four test methods ran against the original backup function; eight subcase failures reproduced alias/case rejection and quoted-password truncation. Canonical/negative controls executed. No real dump or deployment.
|
||||||
|
- Final targeted suite: **5/5 methods PASSED**, including 7 host spellings, quoted semicolon/equal/doubled quote/shell characters, Unicode/single quotes, empty password, and six invalid/ambiguous input cases. Tests run the actual backup function and mock only external dump/Docker boundaries.
|
||||||
|
- Full offline script unittest set: **31/31 PASSED**, including targeted cases, ingress Compose rendering with synthetic env, workflow/runner guards, retry guard, supply-chain fixtures and offline model-evaluation helpers. No model invocation. Printed exit17/exit1 are expected mocked runner negative cases. Do not add targeted five to the overlapping total.
|
||||||
|
- Ordinary unittest discovery found zero tests because repository filenames contain hyphens; recorded as NOT A PASS. Explicit importlib loading of every `scripts/test-*.py` unittest module (excluding standalone Docker topology runner) executed the 31 tests. Docker topology qualification NOT RUN.
|
||||||
|
- Git Bash `bash -n deploy/deploy.sh`: PASSED.
|
||||||
|
- Graphify AST update completed locally: 18,504 nodes / 45,920 edges; missing optional PDF/SQL extractors reported. Newly generated untracked graph output retained locally, not a runtime verification or introduced repository artifact.
|
||||||
|
- Product/backend/frontend/full database/browser suites NOT RUN: this package changes only deploy parsing and its offline tests. Real dump/restore, Linux-host operational behavior, hosted CI and production validation remain NOT TESTED.
|
||||||
|
|
||||||
|
The first Unicode test attempt failed in the Windows test harness encoding before reaching product code; the harness was corrected to UTF-8, then passed. Parser stdin/stdout are explicitly UTF-8. No application user data or production credentials used.
|
||||||
|
|
||||||
|
Rollback: revert this bounded implementation commit to restore the prior parser (and its known limitation). No database/schema change needs reversal. RR-11 is FIXED AND OFFLINE-TESTED on this development branch; RR-12/13 and production remain open. No main merge/deploy authorised.
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
"""Read a deployment connection string from stdin; emit five fields plus a sentinel.
|
||||||
|
|
||||||
|
This is a single-host dump contract, not a replacement for MySqlConnector. Reject
|
||||||
|
ambiguous aliases/control characters instead of choosing a different backup target.
|
||||||
|
Never include input values in errors or pass the connection string as an argument.
|
||||||
|
"""
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
ALIASES = {
|
||||||
|
**dict.fromkeys(['server', 'host', 'data source', 'datasource', 'address', 'addr', 'network address'], 'host'),
|
||||||
|
**dict.fromkeys(['user id', 'userid', 'username', 'uid', 'user name', 'user'], 'user'),
|
||||||
|
**dict.fromkeys(['password', 'pwd'], 'password'),
|
||||||
|
**dict.fromkeys(['database', 'initial catalog'], 'database'),
|
||||||
|
'port': 'port',
|
||||||
|
}
|
||||||
|
PAIR = re.compile(r'''\s*([^=;]+?)\s*=\s*(?:"((?:""|[^"])*)"|'((?:''|[^'])*)'|([^;]*))\s*(?:;|$)''')
|
||||||
|
|
||||||
|
|
||||||
|
def parse(connection):
|
||||||
|
if any(ord(c) < 32 or ord(c) == 127 for c in connection):
|
||||||
|
raise ValueError('Control characters are unsupported in backup connection settings.')
|
||||||
|
fields = {}
|
||||||
|
position = 0
|
||||||
|
while position < len(connection):
|
||||||
|
if connection[position] == ';' or connection[position].isspace():
|
||||||
|
position += 1
|
||||||
|
continue
|
||||||
|
match = PAIR.match(connection, position)
|
||||||
|
if not match:
|
||||||
|
raise ValueError('Malformed backup connection settings.')
|
||||||
|
key, double, single, plain = match.groups()
|
||||||
|
if double is not None:
|
||||||
|
value = double.replace('""', '"')
|
||||||
|
elif single is not None:
|
||||||
|
value = single.replace("''", "'")
|
||||||
|
else:
|
||||||
|
value = plain.strip()
|
||||||
|
if value.startswith(('"', "'")):
|
||||||
|
raise ValueError('Malformed quoted backup connection value.')
|
||||||
|
canonical = ALIASES.get(key.strip().lower())
|
||||||
|
if canonical:
|
||||||
|
if canonical in fields:
|
||||||
|
raise ValueError('Duplicate backup connection setting or alias.')
|
||||||
|
fields[canonical] = value
|
||||||
|
position = match.end()
|
||||||
|
if any(not fields.get(key, '').strip() for key in ['host', 'database', 'user']):
|
||||||
|
raise ValueError('Backup connection requires host, database and user.')
|
||||||
|
port = fields.get('port', '3306')
|
||||||
|
if not re.fullmatch(r'[0-9]+', port) or not 1 <= int(port) <= 65535:
|
||||||
|
raise ValueError('Backup connection port must be between 1 and 65535.')
|
||||||
|
if any(c in fields['host'] for c in ',/\\'):
|
||||||
|
raise ValueError('Backup requires one TCP host, not a host list or socket path.')
|
||||||
|
if any(c in fields['database'] for c in '/\\'):
|
||||||
|
raise ValueError('Database name cannot contain backup path separators.')
|
||||||
|
return [fields['host'], str(int(port)), fields['database'], fields['user'], fields.get('password', '')]
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
try:
|
||||||
|
values = parse(sys.stdin.buffer.read().decode('utf-8'))
|
||||||
|
except (ValueError, UnicodeError):
|
||||||
|
print('Invalid or unsupported backup connection settings.', file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
sys.stdout.reconfigure(encoding='utf-8', newline='\n')
|
||||||
|
# Sentinel keeps an empty final password intact through shell command substitution.
|
||||||
|
sys.stdout.write('\n'.join([*values, 'END']) + '\n')
|
||||||
@@ -0,0 +1,89 @@
|
|||||||
|
"""Exercise the real backup function with a synthetic dump command; never deploy."""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which('bash')
|
||||||
|
|
||||||
|
|
||||||
|
class BackupConnectionTests(unittest.TestCase):
|
||||||
|
def backup(self, connection):
|
||||||
|
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
|
||||||
|
start = source.index('backup_database() {')
|
||||||
|
function = source[start:source.index('\n}\n', start) + 3]
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
folder = Path(temporary)
|
||||||
|
script = '\n'.join([
|
||||||
|
'set -euo pipefail',
|
||||||
|
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
|
||||||
|
'BACKUP_STAMP=synthetic',
|
||||||
|
'DATABASE_PROVIDER=mariadb',
|
||||||
|
'JOBTRACKER_CONNECTION_STRING=' + shlex.quote(connection),
|
||||||
|
'python3() { ' + shlex.quote(Path(sys.executable).as_posix()) + ' "$@"; }',
|
||||||
|
'docker() { echo "Unexpected Docker call" >&2; return 99; }',
|
||||||
|
'mariadb-dump() { printf "%s\\0" "$@" > "$BACKUP_DIR/args"; '
|
||||||
|
'printf "%s" "$MYSQL_PWD" > "$BACKUP_DIR/password"; '
|
||||||
|
'printf "CREATE TABLE synthetic (id int);\\n-- Dump completed\\n"; }',
|
||||||
|
'verify_sql_backup() { gzip -t "$1"; }',
|
||||||
|
function,
|
||||||
|
'backup_database',
|
||||||
|
])
|
||||||
|
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
|
||||||
|
text=True, encoding='utf-8', capture_output=True, cwd=ROOT,
|
||||||
|
env={**os.environ, 'DEPLOY_SKIP_DB_BACKUP': 'false'})
|
||||||
|
arguments = (folder / 'args').read_bytes().split(b'\0')[:-1] if (folder / 'args').exists() else None
|
||||||
|
password = (folder / 'password').read_text(encoding='utf-8') if (folder / 'password').exists() else None
|
||||||
|
return result, arguments, password
|
||||||
|
|
||||||
|
def test_supported_host_alias_and_case_reach_dump_unchanged(self):
|
||||||
|
for host in ['Server', 'SERVER', 'Host', 'Data Source', 'Address', 'Addr', 'Network Address']:
|
||||||
|
with self.subTest(host=host):
|
||||||
|
result, args, password = self.backup(f'{host}=127.0.0.1;DATABASE=synthetic;UID=reader;PWD=dummy-secret')
|
||||||
|
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
|
||||||
|
self.assertEqual([b'--host=127.0.0.1', b'--port=3306', b'--user=reader',
|
||||||
|
b'--single-transaction', b'--routines', b'--events', b'--quick', b'synthetic'], args)
|
||||||
|
self.assertEqual('dummy-secret', password)
|
||||||
|
self.assertNotIn('dummy-secret', result.stdout + result.stderr)
|
||||||
|
|
||||||
|
def test_quoted_separators_and_shell_characters_remain_data(self):
|
||||||
|
result, args, password = self.backup('Server=127.0.0.1;Port=3307;Database=synthetic;User ID=reader;Password="semi;eq=quote""$(false)"')
|
||||||
|
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
|
||||||
|
self.assertIn(b'--port=3307', args)
|
||||||
|
self.assertEqual('semi;eq=quote"$(false)', password)
|
||||||
|
|
||||||
|
def test_empty_password_is_preserved(self):
|
||||||
|
result, args, password = self.backup('Server=127.0.0.1;Database=synthetic;User ID=reader;Password=')
|
||||||
|
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
|
||||||
|
self.assertIsNotNone(args)
|
||||||
|
self.assertEqual('', password)
|
||||||
|
|
||||||
|
def test_single_quoted_unicode_password_and_database_alias_are_preserved(self):
|
||||||
|
result, args, password = self.backup("Host=127.0.0.1;Initial Catalog=synthetic;Username=reader;Password='søknad你好;it''s';SslMode=VerifyFull")
|
||||||
|
self.assertEqual(0, result.returncode, result.stderr + result.stdout)
|
||||||
|
self.assertEqual(b'synthetic', args[-1])
|
||||||
|
self.assertEqual("søknad你好;it's", password)
|
||||||
|
|
||||||
|
def test_invalid_or_ambiguous_input_never_invokes_dump_or_exposes_input(self):
|
||||||
|
for connection in [
|
||||||
|
'Host=127.0.0.1;Database=synthetic;Uid=reader;Port=0;Pwd=dummy-secret',
|
||||||
|
'Host=127.0.0.1;Database=synthetic;Uid=reader;Port=65536;Pwd=dummy-secret',
|
||||||
|
'Host=127.0.0.1;Server=other;Database=synthetic;Uid=reader;Pwd=dummy-secret',
|
||||||
|
'Host=127.0.0.1;Database=synthetic;Uid=reader;Pwd="dummy-secret',
|
||||||
|
'Host=127.0.0.1;Database=synthetic;Uid=reader;Pwd=dummy-secret\nline',
|
||||||
|
'Host=127.0.0.1;Uid=reader;Pwd=dummy-secret',
|
||||||
|
]:
|
||||||
|
with self.subTest(connection=connection.replace('dummy-secret', 'REDACTED')):
|
||||||
|
result, args, _ = self.backup(connection)
|
||||||
|
self.assertNotEqual(0, result.returncode)
|
||||||
|
self.assertIsNone(args)
|
||||||
|
self.assertNotIn('dummy-secret', result.stdout + result.stderr)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user