Compare commits

...

5 Commits

21 changed files with 1270 additions and 40 deletions
+20 -21
View File
@@ -426,10 +426,9 @@ verify_sql_backup() {
local has_schema=1 has_trailer=1
set +o pipefail
if gzip -dc "$target" | grep -q "CREATE TABLE"; then has_schema=0; fi
# mariadb-dump / mysqldump write "-- Dump completed on ..." as the last line.
# Its absence means the dump died partway through.
if gzip -dc "$target" | grep -q "Dump completed"; then has_trailer=0; fi
set -o pipefail
# Require the final nonblank completion comment, not that phrase in SQL data.
if gzip -dc "$target" | awk '{ sub(/\r$/, "") } NF { last=$0 } END { exit !(last ~ /^-- Dump completed( on .*)?$/) }'; then has_trailer=0; fi
if [ "$has_schema" -ne 0 ]; then
echo "Backup ${target} contains no CREATE TABLE statement — it is not a usable dump. Aborting deploy."
@@ -454,19 +453,15 @@ verify_volume_backup() {
# Prove this is the DATABASE volume, not merely a non-empty tar. This is what
# catches an archive taken from the wrong volume, or from a freshly created
# empty one.
# Listed in the same container that wrote it, not with the host's tar: GNU tar
# reads a leading "C:/" as an rsh host and fails outright when this script runs
# from Git Bash. Docker is already required to have produced the archive.
# Python is already required by deployment validation. Its tar reader handles
# Windows paths and checks regular-file/header bytes without extracting files.
local has_db=1
if docker run --rm \
-v "$(dirname "$target")":/backup:ro \
-e ARCHIVE_NAME="$(basename "$target")" \
alpine:3 sh -c 'tar tzf "/backup/$ARCHIVE_NAME" > /tmp/backup-members && grep -Eq "(^|/)jobtracker[.]db$" /tmp/backup-members' >/dev/null 2>&1; then
if python3 scripts/validate-sqlite-backup.py "$target"; then
has_db=0
fi
if [ "$has_db" -ne 0 ]; then
echo "Archive ${target} contains no jobtracker.db — it is not a SQLite data backup. Aborting deploy."
echo "Archive ${target} has no valid regular SQLite database member. Aborting deploy."
rm -f "$target"
return 1
fi
@@ -522,14 +517,12 @@ backup_recovery_volumes() {
apply_backup_retention() {
local daily_days="${BACKUP_DAILY_RETENTION_DAYS:-30}"
local monthly_count="${BACKUP_MONTHLY_RETENTION_COUNT:-12}"
local timezone="${BACKUP_TIMEZONE:-Europe/Oslo}"
if [[ ! "$daily_days" =~ ^[1-9][0-9]*$ ]] || [[ ! "$monthly_count" =~ ^[1-9][0-9]*$ ]]; then
echo "Backup retention must use positive integer day/count values."
return 1
fi
local month monthly_root monthly_dir complete_tmp
month="$(TZ="$timezone" date +%Y-%m)"
local month="$1" monthly_root monthly_dir complete_tmp
monthly_root="$BACKUP_DIR/monthly"
monthly_dir="$monthly_root/$month"
if ! mkdir -p "$monthly_root" || ! chmod 700 "$monthly_root"; then
@@ -610,12 +603,14 @@ upload_current_backup_set() {
fi
done
local month="$(TZ="${BACKUP_TIMEZONE:-Europe/Oslo}" date +%Y-%m)"
if [ -d "$BACKUP_DIR/monthly/$month" ]; then
if ! rclone copy "$BACKUP_DIR/monthly/$month" "$remote/monthly/$month" --immutable; then
echo "Monthly off-host backup upload failed. Aborting deploy."
return 1
fi
local month="$1"
if [ ! -f "$BACKUP_DIR/monthly/$month/.complete" ]; then
echo "Completed monthly recovery set is missing. Aborting deploy."
return 1
fi
if ! rclone copy "$BACKUP_DIR/monthly/$month" "$remote/monthly/$month" --immutable; then
echo "Monthly off-host backup upload failed. Aborting deploy."
return 1
fi
echo "Current recovery set copied to configured off-host remote."
}
@@ -643,7 +638,11 @@ if ! backup_recovery_volumes; then
exit 1
fi
if ! apply_backup_retention || ! upload_current_backup_set; then
if ! backup_month="$(TZ="${BACKUP_TIMEZONE:-Europe/Oslo}" date +%Y-%m)"; then
echo "Could not determine the monthly recovery period. Aborting deploy."
exit 1
fi
if ! apply_backup_retention "$backup_month" || ! upload_current_backup_set "$backup_month"; then
exit 1
fi
@@ -1,5 +1,17 @@
# Recovery implementation review and rehearsal plan
## RR-13 monthly recovery period across retention and upload
19 November 2026, isolated release branch from `67b9b675194d3c26432a4da156f680d3368f767e`. A backup that crossed the local month boundary between retention and upload could omit the retained monthly set while reporting success. A failed calendar command could select the monthly root as the destination. A monthly set that disappeared before transfer could also produce a success message. All three cases were reproduced with synthetic files and a recording transfer stub.
The gate now resolves the month once in the configured backup timezone, checks the command status and passes that same month to both operations. Upload requires the retained snapshot's completion marker. A previously completed snapshot for the month retains its existing stamp and contents.
Final verification passed 33 backup methods (13.456 seconds), six rollback methods (0.380 seconds) and Bash syntax. One intermediate test run misplaced a pre-existing success assertion in a failure case. The assertion was restored to the successful upload case before the final run. The upload harness executes the real retention and upload functions over temporary files with pruning and network transfer stubbed. No off-host copy, remote encryption, recovery rehearsal or deployment was performed. Reverting this change affects script behaviour only.
## RR-17 SQLite archive member format
18 November2026, isolated source10abd22f from2fb10f7b. Name-only verification accepted ten synthetic link/type/content/path cases that could not establish database bytes. Final29 backup methods and Bash syntax pass with the streaming Python standard-library path/type/signature verifier. No member is extracted. Supported root/dot/nested paths remain; Python3 was already a mandatory deployment dependency. Final native logs and precise intermediate fixture failures are retained in [format evidence](evidence/sqlite-archive-format/README.md). Earlier26/22/21 counts overlap29. This is not SQLite integrity, authoritative path selection, WAL/key/tombstone consistency, restore/RPO/RTO, actual container or production qualification. Original release worktree and A3 remain separate.
## RR-16 tar failure propagation
16 November 2026. Source `a516bfb4b38468f9bdc222744a02c07bc5b013c3`, after RR-15 on `codex/release-rollback-health`. A valid gzip stream containing a tar database header without its declared body passed verification: tar failed after printing the matching member and the old pipeline returned grep's success. One deterministic local fixture reproduced this false acceptance and next-gate continuation.
@@ -20,5 +20,6 @@ Discovery only, 2026-11-02. These are qualification gaps unless a concrete defec
| RR-14 | July first-deploy runbook inferred rollback safety from its no-op migration baseline, while current migration Up methods add schema | Historical scope and current qualification links clarified. `AddJobRevision` adds Version and `AddUserOperationEvents` creates a table/indexes. Exact old/new image compatibility after migration/writes remains an isolated rehearsal gate |
| RR-15 | SQLite backup listing accepted a WAL or similarly named file without the database member | FIXED AND OFFLINE-TESTED at `4c6c2207`: exact member basename required; three false-positive archives reproduced first,21 backup methods/syntax passed. [Bounded evidence](11_BACKUP_RESTORE_ROLLBACK.md#rr-15-sqlite-archive-member-check). Real SQLite contents, regular-file/link safety, tar errors, restore and recovery-set consistency remain unqualified |
| RR-16 | Tar failure was hidden by a matching member's successful grep | FIXED AND OFFLINE-TESTED at `a516bfb4`: valid gzip/truncated tar reproduced false acceptance; tar listing must succeed before matching.22 backup methods/syntax passed; [bounded evidence](11_BACKUP_RESTORE_ROLLBACK.md#rr-16-tar-failure-propagation). Real target-image/restore/security qualification remains open |
| RR-17 | SQLite archive name-only checks accepted links/special entries, non-database bytes and unsafe database paths | FIXED AND OFFLINE-TESTED at `10abd22f`: streaming stdlib validation of relative path, regular member and SQLite signature. Ten false-acceptance subtests reproduced first; final29 backup methods/syntax passed. [Evidence and limits](evidence/sqlite-archive-format/README.md). Header-only format is not integrity/restore/WAL/recovery-set proof; actual host/container/release qualification remains open |
Continue evidence work against the [source endpoint and environment census](03_ENDPOINT_ENVIRONMENT_CENSUS.md). Actual qualification requires an accepted candidate and isolated environment. Every fix needs minimal scope, targeted verification and the applicable stage approval.
+2
View File
@@ -1,5 +1,7 @@
# Release discovery handoff
18 November2026: isolated codex/recovery-archive-validation from2fb10f7b, source10abd22f. RR-17 streaming SQLite archive member path/type/header validation closes ten reproduced false acceptances;29 offline backup methods/syntax pass. No real backup/restore/container/deployment. [Evidence](evidence/sqlite-archive-format/README.md) retains original failures, Windows test-fixture correction and remaining recovery limits. Original release worktree is untouched; no A3 integration or release acceptance.
11 November. Branch `codex/release-rollback-health`, baseline `24a62997`, source `a39ac847` adds bounded post-rollback frontend/liveness/readiness probes and failing-diagnostics coverage while preserving the original exit status. Six extracted-handler tests, twelve backup tests and shell syntax validation passed. Full deploy, Docker, network, models and real restore were not executed. [Evidence and remaining qualification](11_BACKUP_RESTORE_ROLLBACK.md#rr-12-post-rollback-core-health-checks).
RR-14 clarifies that the first-production runbook is a July baseline. Current Up migrations add the Version column and operation-event schema. Its old no-op argument cannot qualify a current old/new image pair. Current candidate, migration/write and recovery rehearsal requirements are now linked from that runbook. This was source/document review only.
@@ -0,0 +1,39 @@
# Offline SQL completion-marker selection
18 November 2026. Branch `codex/release-offline-recovery`, baseline `2fb41a96ce51847ac2ed2d1c9b2e08d8a2ff1129` recovered from `codex/release-rollback-health`. Original checkout/source was left unchanged; its existing untracked graph was preserved. The full authoritative `E:/Storage/Projects/Active/temp docs/final/prompt 11 - release readiness.txt` was read. Sections 13 and 16 permit safe offline investigation and focused remediation; they do not grant live recovery or release permission. Existing release handoff, recovery report, remediation register and acceptance matrix retain their gates.
## Reproduced defect
The real `verify_sql_backup` function described a required final dump trailer, but accepted the substring `Dump completed` anywhere in decompressed SQL. Two valid-gzip, incomplete synthetic dumps passed and reached the next mocked deployment gate:
- A `CREATE TABLE` followed by `INSERT INTO synthetic VALUES ('Dump completed');`, with no completion comment.
- A completion comment followed by another SQL statement, so that comment was not a final trailer.
Before changing the shell source, the four new test methods reported **two assertion failures and two passing controls**. Both negative cases returned exit 0 and printed `FORMAT VERIFIED` and `NEXT DEPLOY GATE WOULD START`. This is an offline false-acceptance defect, not evidence that a real deployed backup is damaged. Engineering priority proposed P2; no release-risk acceptance or security severity is implied.
## Minimal change
The verifier now checks the final nonblank line for a completion comment, using the existing host awk dependency. CRLF is normalised for this check; trailing blank/whitespace-only lines remain allowed. Both the actual timestamped `-- Dump completed on ...` form and the short `-- Dump completed` fixture form remain accepted. Existing pipefail is restored before this full-stream check, so a decompression/pipeline failure cannot be hidden by its last command. The schema predicate, permissions, retention, archive verification, transfer, tombstone handling and rollback code are unchanged.
The tests run only extracted `verify_backup_file_shape`/`verify_sql_backup` functions under the real conditional caller shape. Real local gzip creates/tests/decompresses synthetic files; real Git Bash/awk evaluate the predicate. The new fixture stubs the invalid-artifact removal and final report boundaries. It never sources or executes the complete deployment script. It creates no SQL database and performs no restore, container operation, network transfer, pruning or live deletion. Python removes only its disposable synthetic temporary directories. Temp/output were kept in the isolated E-drive worktree.
## Verification
```powershell
rtk proxy python scripts/test-deploy-backup.py BackupSqlTrailerTests -v
rtk proxy python scripts/test-deploy-backup.py -v
rtk proxy "C:/Program Files/Git/bin/bash.exe" --noprofile --norc -n deploy/deploy.sh
rtk git diff --check
```
- RED: four new methods; two expected failures, two positive/negative controls passed.
- GREEN: **26/26 backup methods passed**, including four new methods. The 22 inherited methods were rerun because the changed verifier also participates in the existing permission/backup-gate fixtures. Do not add the overlapping four-method or historical 22-method runs to this total. Native output: [green-backup.txt](green-backup.txt).
- Bash syntax: exit 0, no stdout/stderr. Source identities and native log SHA-256: [SOURCE_RECEIPTS.json](SOURCE_RECEIPTS.json).
- Whitespace check passed. No new package, dependency installation, app/.NET build, browser or server lane was used.
- Graph-first query used the existing release graph before source inspection. It was copied as an isolated update seed, then `graphify update . --no-cluster` ran AST-only here to avoid clustering CPU during the separate full-Jest lane. No semantic/model extraction or shared graph mutation was requested. Generated graph stays untracked local navigation material; optional extractor warnings are not runtime evidence.
## Remaining gates and rollback
This validates completion-marker selection only. It is not a SQL parser, usable-data proof, consistent recovery-set verification, cryptographic authenticity check, Linux-host qualification or restore rehearsal. A syntactically plausible completion comment alone cannot prove the SQL is executable or complete. Actual dump-tool/version output, encrypted off-host retrieval/decryption, permissions, database/files/keys/tombstone consistency, post-migration/post-write rollback and agreed RPO/RTO still require authorised isolated qualification. RR-09/RR-12/RR-13 and production NO-GO remain open.
The six rollback methods were not rerun because that handler is unchanged. Product, MariaDB, container, provider, hosted CI, off-host, real-model and production checks were not run. Root owns publication/integration and central records. Reverting this bounded source commit restores the previous false-acceptance behaviour; there is no database/schema/data migration to reverse.
@@ -0,0 +1,23 @@
{
"baseline": "2fb41a96ce51847ac2ed2d1c9b2e08d8a2ff1129",
"scope": "offline SQL completion-marker selection only; no recovery qualification",
"python": "3.12.10 (tags/v3.12.10:0cc8128, Apr 8 2025, 12:21:36) [MSC v.1943 64 bit (AMD64)]",
"sourceSha256": {
"deploy\\deploy.sh": "a86c532f6689e386ab68bd49322718abd4e303bd70b450e990d7ce8106cd2132",
"scripts\\test-deploy-backup.py": "a6f9e5e3523c45a898e0a340f64cbcb3d06cc8f1873e249983ad513125f4886b"
},
"bashSyntax": {
"command": [
"C:/Program Files/Git/bin/bash.exe",
"--noprofile",
"--norc",
"-n",
"deploy/deploy.sh"
],
"exitCode": 0,
"stdout": "",
"stderr": ""
},
"testReceiptSha256": "643fe0e73c222f33d98465c070d0173332558bbca244341fee20b64e8ed05934",
"productDatabaseContainerNetworkRestoreTests": "NOT RUN"
}
@@ -0,0 +1,31 @@
test_empty_password_is_preserved (__main__.BackupConnectionTests.test_empty_password_is_preserved) ... ok
test_invalid_or_ambiguous_input_never_invokes_dump_or_exposes_input (__main__.BackupConnectionTests.test_invalid_or_ambiguous_input_never_invokes_dump_or_exposes_input) ... ok
test_quoted_separators_and_shell_characters_remain_data (__main__.BackupConnectionTests.test_quoted_separators_and_shell_characters_remain_data) ... ok
test_single_quoted_unicode_password_and_database_alias_are_preserved (__main__.BackupConnectionTests.test_single_quoted_unicode_password_and_database_alias_are_preserved) ... ok
test_supported_host_alias_and_case_reach_dump_unchanged (__main__.BackupConnectionTests.test_supported_host_alias_and_case_reach_dump_unchanged) ... ok
test_exact_database_member_at_root_or_in_a_directory_is_accepted (__main__.BackupDatabaseMemberTests.test_exact_database_member_at_root_or_in_a_directory_is_accepted) ... ok
test_sidecar_or_similar_name_is_not_the_database_member (__main__.BackupDatabaseMemberTests.test_sidecar_or_similar_name_is_not_the_database_member) ... ok
test_valid_gzip_with_truncated_tar_database_member_is_rejected (__main__.BackupDatabaseMemberTests.test_valid_gzip_with_truncated_tar_database_member_is_rejected) ... ok
test_both_permission_failures_block_verification_and_next_gate (__main__.BackupPermissionTests.test_both_permission_failures_block_verification_and_next_gate) ... ok
test_fallback_permission_success_preserves_verification (__main__.BackupPermissionTests.test_fallback_permission_success_preserves_verification) ... ok
test_host_permission_success_does_not_need_fallback (__main__.BackupPermissionTests.test_host_permission_success_does_not_need_fallback) ... ok
test_completed_monthly_snapshot_contains_both_artifacts_and_stamp (__main__.BackupRetentionTests.test_completed_monthly_snapshot_contains_both_artifacts_and_stamp) ... ok
test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_daily_prune_failure_blocks_monthly_pruning_upload_and_deploy) ... ok
test_empty_monthly_enumeration_does_not_request_removal (__main__.BackupRetentionTests.test_empty_monthly_enumeration_does_not_request_removal) ... ok
test_marker_publish_failure_does_not_expose_complete_marker_or_proceed (__main__.BackupRetentionTests.test_marker_publish_failure_does_not_expose_complete_marker_or_proceed) ... ok
test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_enumeration_failure_blocks_pruning_upload_and_deploy) ... ok
test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy (__main__.BackupRetentionTests.test_monthly_prune_failure_stops_before_remaining_months_upload_and_deploy) ... ok
test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed (__main__.BackupRetentionTests.test_partial_monthly_copy_cannot_mark_snapshot_complete_or_proceed) ... ok
test_preserves_the_existing_completed_monthly_snapshot (__main__.BackupRetentionTests.test_preserves_the_existing_completed_monthly_snapshot) ... ok
test_completion_comment_before_more_sql_is_not_a_final_trailer (__main__.BackupSqlTrailerTests.test_completion_comment_before_more_sql_is_not_a_final_trailer) ... ok
test_completion_phrase_in_sql_value_is_not_a_dump_trailer (__main__.BackupSqlTrailerTests.test_completion_phrase_in_sql_value_is_not_a_dump_trailer) ... ok
test_final_completion_comment_is_accepted_with_supported_line_endings (__main__.BackupSqlTrailerTests.test_final_completion_comment_is_accepted_with_supported_line_endings) ... ok
test_missing_completion_comment_remains_rejected (__main__.BackupSqlTrailerTests.test_missing_completion_comment_remains_rejected) ... ok
test_daily_copy_failure_stops_before_remaining_files_and_monthly_copy (__main__.BackupUploadTests.test_daily_copy_failure_stops_before_remaining_files_and_monthly_copy) ... ok
test_monthly_copy_failure_blocks_deploy (__main__.BackupUploadTests.test_monthly_copy_failure_blocks_deploy) ... ok
test_uploads_only_current_daily_files_then_monthly_snapshot (__main__.BackupUploadTests.test_uploads_only_current_daily_files_then_monthly_snapshot) ... ok
----------------------------------------------------------------------
Ran 26 tests in 10.986s
OK
@@ -0,0 +1,27 @@
# RR-17: SQLite archive database member validation
Baseline2fb10f7b, isolated codex/recovery-archive-validation. The old name-only tar listing accepted symlinks/hardlinks/FIFO/device entries, empty or unrelated bytes named jobtracker.db, and database paths outside the archive root. Ten negative subtests freshly reproduced successful verification and continuation to the mocked next deployment gate.
The existing deployment verifier now calls a small Python standard-library streaming tar reader. Python3 is already required by the mandatory ingress validator. It checks each matching database basename for a safe relative path, regular-file type and the16-byte SQLite signature; it reads through the archive without extracting members. Root, dot-prefixed and nested supported database names remain accepted. Real SQLite test fixtures are created with stdlib sqlite3. No package, database/schema, production configuration or persistent data is changed by these tests.
- Final29/29 backup methods PASSED,0skips; includes earlier26 and three new methods/ten negative subtests. Bash syntax/whitespace PASSED. Existing dump/parser, upload, retention and permission checks remain. Real local gzip and Python tar/SQLite fixture creation were exercised; no Docker, actual backup, recovery or deployment.
- Initial red fixture had14 Windows cleanup errors because a context-managed sqlite connection had not been closed. After explicitly closing it, corrected red had10 failures and0errors; these are the valid application defect reproductions. Original draft errors are retained and are not product defects.
- First whole-suite green exposed three inherited permission-fixture failures: those tests mocked the old Docker validator over arbitrary bytes. Recovery permission fixtures now use a genuine gzip/tar with a format header and the real Python validator, preserving byte-equality/no-removal/next-gate assertions. Final29 rerun adds a Docker trap in member tests to prove no container validation occurs. Intermediate results overlap29.
- Exact native logs/gzip plus final source hashes: [receipts](SOURCE_RECEIPTS.json). Original production baseline is bound. Red test fixture bytes were not frozen at execution; don't claim an immutable full red-test snapshot. Test helper changes after red and all intermediate failures are documented.
This checks **format**, not SQLite integrity or usefulness: a signature-only regular file satisfies this bounded predicate. It does not validate arbitrary other archive members, select an authoritative database among multiple compatible paths, prove WAL consistency, restore records, verify keys/tombstones/off-host encryption, qualify RPO/RTO or replace recovery rehearsal. MariaDB's dump path is unchanged. Actual Linux/container/host deployment remains NOT RUN. No release/security approval is granted.
Rollback restores verify_volume_backup and removes the helper/associated fixture changes on this isolated branch. No data rollback is required. Original release worktree and A3 application branch remain untouched by this package.
## Host archive producer compatibility, 19 November 2026
The test at `BackupDatabaseMemberTests.test_host_tar_producer_with_database_and_sidecar` uses the deployment command's `tar czf … -C data .` arguments with a temporary synthetic volume. It creates a real SQLite database with one row and closes the connection. The host tar command archives it alongside an inert file named `jobtracker.db-wal`. The production Python validator accepts the archive, and a read-only comparison confirms the database member matches the source bytes. A second archive containing only the sidecar name is rejected.
| Windows producer | Verification | Result |
| --- | --- | --- |
| Git for Windows GNU tar 1.35 | Full backup suite, Python 3.12.10 / SQLite 3.49.1 | 30 methods passed in 12.486 seconds |
| Windows bsdtar 3.8.9 / libarchive 3.8.9 | Focused producer method with the same positive and negative cases | Passed in 0.192 seconds |
The focused run overlaps the full suite. To repeat, put the desired tar directory first on PATH and run `python scripts/test-deploy-backup.py BackupDatabaseMemberTests.test_host_tar_producer_with_database_and_sidecar -v`. The test resolves the absolute producer path before invocation. Source baseline `5718d9521be0935e1c0364a775ed8ce7d0702f43`. The validator and deployment script are unchanged.
Linux/Alpine producer qualification remains blocked on a pre-existing isolated Linux environment. WSL listed only a stopped Docker Desktop distribution. This fixture's sidecar exercises filename classification only. Live WAL consistency, restore, deployment, permissions, encryption and recovery-set completeness remain open.
@@ -0,0 +1,59 @@
{
"baseline": "2fb10f7b4e1968e40158773aff68b9136b6366b6",
"sources": [
{
"file": "deploy/deploy.sh",
"sha256Lf": "f0b23f3a94ce62df037995e24514806e0dfe390d8bab83cc98f0006c004b7bf3"
},
{
"file": "scripts/test-deploy-backup.py",
"sha256Lf": "b5d78c98cb3fac20e6a5a01de45ffaed6c039445f04e9af4b6e4fdca5a5cd763"
},
{
"file": "scripts/validate-sqlite-backup.py",
"sha256Lf": "b643ca6997b735014634efc64e47175e7e94a8d2e26efce58a15c30af45dadbb"
}
],
"nativeArtifacts": [
{
"file": "archive-red.txt",
"rawSha256": "b6b74f520908b9dbdab1795394e6b060418fa81d6960bdcc95e57618b6761f7e",
"gzipSha256": "da9cdc67fd3788f5fee37116cc3f28f172e6af41f64018f1801071df49a14d47",
"canonicalSha256": "8fbe782bcb534264b8ec67c78b4d4f13b93bb799313267f76985f3398c1d149a"
},
{
"file": "archive-red-corrected.txt",
"rawSha256": "f609f90035c2ad3a6ad892cc1889b899c64d40e38b5d76b4e0d2b5f0f4fd5296",
"gzipSha256": "8b88d44088d8c2414b9568d669ab7aca03c81397ede832aa70e24c6dbe7e445e",
"canonicalSha256": "e975026e58575724c8de19997a6754023da8b4f390772b4a3c0eedd060eb2682"
},
{
"file": "archive-green.txt",
"rawSha256": "099a5507befb5a39b24925a56974b35ae022926012297c8bc51b2ee8c9f8ecac",
"gzipSha256": "82a57389cdcd7753ceea35b88ca03bc4db9b592a46cca65ae319e1f296e05bde",
"canonicalSha256": "cd869864c07943aae4c87105d9b352dce9fedacc41b1a64516f27bce08696807"
},
{
"file": "archive-green-final.txt",
"rawSha256": "fe75805217c87fe7e31601ac226015c754a385fab40e81bb32569b88ea69bdc7",
"gzipSha256": "ffa1a1f6e34c4acff97476abd315fd8d5cd81046436ca3b781fed7f3fd2da335",
"canonicalSha256": "55ac699218412a437daaeb3b0d6de51cbcb21809a2b9d1a81fa95b08e3160a39"
},
{
"file": "archive-qualified.txt",
"rawSha256": "be26b9a73900d322f4f6b417ad9e7d1800e5d71b42e5809b37b51adf3a383252",
"gzipSha256": "aad2d2295d57a6d3ac8b7f8d89b816f8f959ce14a78bd9fed573504ec2fb9d8a",
"canonicalSha256": "5f397b90c78a3fecde2eed7a3a41d8549cf70944d4ea0e0d0e04ec14630a4d2a"
}
],
"checks": {
"backupMethods": 29,
"backupExitCode": 0,
"bashSyntaxExitCode": 0
},
"limits": [
"SQLite header/type/path format only; no integrity check/restore/container/recovery acceptance",
"Red test fixture bytes were not frozen at execution; original production baseline and native failure logs retained."
],
"baselineDeployBlob": "77a5781ff22b2b7fc11cef3d3969b0f47bb9dded"
}
@@ -0,0 +1,11 @@
python : .............................
At line:1 char:124
+ ... =$env:TEMP; python scripts/test-deploy-backup.py *> ".tmp/archive-gre ...
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : NotSpecified: (.............................:String) [], RemoteException
+ FullyQualifiedErrorId : NativeCommandError
----------------------------------------------------------------------
Ran 29 tests in 15.186s
OK
@@ -0,0 +1,62 @@
python : ...........FFF...............
At line:1 char:124
+ ... =$env:TEMP; python scripts/test-deploy-backup.py *> ".tmp/archive-gre ...
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : NotSpecified: (...........FFF...............:String) [], RemoteException
+ FullyQualifiedErrorId : NativeCommandError
======================================================================
FAIL: test_both_permission_failures_block_verification_and_next_gate
(__main__.BackupPermissionTests.test_both_permission_failures_block_verification_and_next_gate) (kind='recovery')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 358, in
test_both_permission_failures_block_verification_and_next_gate
result = self.permission_gate(kind, 1, 23)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 334, in permission_gate
self.assertNotIn('UNEXPECTED REMOVAL', result.stdout)
AssertionError: 'UNEXPECTED REMOVAL' unexpectedly found in 'Archive E:/Storage/Projects/Active/temp
docs/jobjakt-local-build/recovery-archive-validation/.tmp/backup-permission-wekbjtnf/synthetic-backup.gz has no valid
regular SQLite database member. Aborting deploy.\nUNEXPECTED REMOVAL\n'
======================================================================
FAIL: test_fallback_permission_success_preserves_verification
(__main__.BackupPermissionTests.test_fallback_permission_success_preserves_verification) (kind='recovery')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 349, in
test_fallback_permission_success_preserves_verification
result = self.permission_gate(kind, 1, 0)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 334, in permission_gate
self.assertNotIn('UNEXPECTED REMOVAL', result.stdout)
AssertionError: 'UNEXPECTED REMOVAL' unexpectedly found in 'Archive E:/Storage/Projects/Active/temp
docs/jobjakt-local-build/recovery-archive-validation/.tmp/backup-permission-56pyov4f/synthetic-backup.gz has no valid
regular SQLite database member. Aborting deploy.\nUNEXPECTED REMOVAL\n'
======================================================================
FAIL: test_host_permission_success_does_not_need_fallback
(__main__.BackupPermissionTests.test_host_permission_success_does_not_need_fallback) (kind='recovery')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 340, in
test_host_permission_success_does_not_need_fallback
result = self.permission_gate(kind, 0, 23)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 334, in permission_gate
self.assertNotIn('UNEXPECTED REMOVAL', result.stdout)
AssertionError: 'UNEXPECTED REMOVAL' unexpectedly found in 'Archive E:/Storage/Projects/Active/temp
docs/jobjakt-local-build/recovery-archive-validation/.tmp/backup-permission-fwk2luv_/synthetic-backup.gz has no valid
regular SQLite database member. Aborting deploy.\nUNEXPECTED REMOVAL\n'
----------------------------------------------------------------------
Ran 29 tests in 17.076s
FAILED (failures=3)
@@ -0,0 +1,11 @@
python : .............................
At line:1 char:124
+ ... =$env:TEMP; python scripts/test-deploy-backup.py *> ".tmp/archive-qua ...
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : NotSpecified: (.............................:String) [], RemoteException
+ FullyQualifiedErrorId : NativeCommandError
----------------------------------------------------------------------
Ran 29 tests in 13.104s
OK
@@ -0,0 +1,144 @@
python : FFF.FFFF.FFF.
At line:1 char:124
+ ... =$env:TEMP; python scripts/test-deploy-backup.py BackupDatabaseMember ...
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : NotSpecified: (FFF.FFFF.FFF.:String) [], RemoteException
+ FullyQualifiedErrorId : NativeCommandError
======================================================================
FAIL: test_empty_or_non_sqlite_database_bytes_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_empty_or_non_sqlite_database_bytes_block_the_next_gate) (contents=b'')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 499, in
test_empty_or_non_sqlite_database_bytes_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_empty_or_non_sqlite_database_bytes_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_empty_or_non_sqlite_database_bytes_block_the_next_gate) (contents=b'unrelated
application bytes')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 499, in
test_empty_or_non_sqlite_database_bytes_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_empty_or_non_sqlite_database_bytes_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_empty_or_non_sqlite_database_bytes_block_the_next_gate) (contents=b'SQLite
format 3X\x00')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 499, in
test_empty_or_non_sqlite_database_bytes_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_links_and_special_members_do_not_substitute_for_database_bytes
(__main__.BackupDatabaseMemberTests.test_links_and_special_members_do_not_substitute_for_database_bytes) (kind=b'2')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 491, in
test_links_and_special_members_do_not_substitute_for_database_bytes
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_links_and_special_members_do_not_substitute_for_database_bytes
(__main__.BackupDatabaseMemberTests.test_links_and_special_members_do_not_substitute_for_database_bytes) (kind=b'1')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 491, in
test_links_and_special_members_do_not_substitute_for_database_bytes
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_links_and_special_members_do_not_substitute_for_database_bytes
(__main__.BackupDatabaseMemberTests.test_links_and_special_members_do_not_substitute_for_database_bytes) (kind=b'6')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 491, in
test_links_and_special_members_do_not_substitute_for_database_bytes
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_links_and_special_members_do_not_substitute_for_database_bytes
(__main__.BackupDatabaseMemberTests.test_links_and_special_members_do_not_substitute_for_database_bytes) (kind=b'3')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 491, in
test_links_and_special_members_do_not_substitute_for_database_bytes
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_unsafe_database_member_paths_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_unsafe_database_member_paths_block_the_next_gate) (name='../jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 507, in
test_unsafe_database_member_paths_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_unsafe_database_member_paths_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_unsafe_database_member_paths_block_the_next_gate) (name='/jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 507, in
test_unsafe_database_member_paths_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_unsafe_database_member_paths_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_unsafe_database_member_paths_block_the_next_gate)
(name='data/../../jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 507, in
test_unsafe_database_member_paths_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
----------------------------------------------------------------------
Ran 6 tests in 3.674s
FAILED (failures=10)
@@ -0,0 +1,619 @@
python : FFFEEEEEEEEEEEEEE
At line:1 char:182
+ ... | Out-Null; python scripts/test-deploy-backup.py BackupDatabaseMember ...
+ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ CategoryInfo : NotSpecified: (FFFEEEEEEEEEEEEEE:String) [], RemoteException
+ FullyQualifiedErrorId : NativeCommandError
======================================================================
ERROR: test_exact_database_member_at_root_or_in_a_directory_is_accepted
(__main__.BackupDatabaseMemberTests.test_exact_database_member_at_root_or_in_a_directory_is_accepted)
(member='jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpxsvw39ie\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 475, in
test_exact_database_member_at_root_or_in_a_directory_is_accepted
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpxsvw39ie\\fixture.db'
======================================================================
ERROR: test_exact_database_member_at_root_or_in_a_directory_is_accepted
(__main__.BackupDatabaseMemberTests.test_exact_database_member_at_root_or_in_a_directory_is_accepted)
(member='./jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpxal3zgni\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 475, in
test_exact_database_member_at_root_or_in_a_directory_is_accepted
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpxal3zgni\\fixture.db'
======================================================================
ERROR: test_exact_database_member_at_root_or_in_a_directory_is_accepted
(__main__.BackupDatabaseMemberTests.test_exact_database_member_at_root_or_in_a_directory_is_accepted)
(member='data/jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpbf17iu6o\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 475, in
test_exact_database_member_at_root_or_in_a_directory_is_accepted
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpbf17iu6o\\fixture.db'
======================================================================
ERROR: test_links_and_special_members_do_not_substitute_for_database_bytes
(__main__.BackupDatabaseMemberTests.test_links_and_special_members_do_not_substitute_for_database_bytes) (kind=b'2')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpp54g45zx\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 489, in
test_links_and_special_members_do_not_substitute_for_database_bytes
result = self.verify_archive('jobtracker.db', member_type=kind)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpp54g45zx\\fixture.db'
======================================================================
ERROR: test_links_and_special_members_do_not_substitute_for_database_bytes
(__main__.BackupDatabaseMemberTests.test_links_and_special_members_do_not_substitute_for_database_bytes) (kind=b'1')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpr68bdxgv\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 489, in
test_links_and_special_members_do_not_substitute_for_database_bytes
result = self.verify_archive('jobtracker.db', member_type=kind)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpr68bdxgv\\fixture.db'
======================================================================
ERROR: test_links_and_special_members_do_not_substitute_for_database_bytes
(__main__.BackupDatabaseMemberTests.test_links_and_special_members_do_not_substitute_for_database_bytes) (kind=b'6')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpvy0f3suo\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 489, in
test_links_and_special_members_do_not_substitute_for_database_bytes
result = self.verify_archive('jobtracker.db', member_type=kind)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpvy0f3suo\\fixture.db'
======================================================================
ERROR: test_links_and_special_members_do_not_substitute_for_database_bytes
(__main__.BackupDatabaseMemberTests.test_links_and_special_members_do_not_substitute_for_database_bytes) (kind=b'3')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpgd_8ky4z\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 489, in
test_links_and_special_members_do_not_substitute_for_database_bytes
result = self.verify_archive('jobtracker.db', member_type=kind)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpgd_8ky4z\\fixture.db'
======================================================================
ERROR: test_sidecar_or_similar_name_is_not_the_database_member
(__main__.BackupDatabaseMemberTests.test_sidecar_or_similar_name_is_not_the_database_member)
(member='jobtracker.db-wal')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpqyiwrtov\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 467, in
test_sidecar_or_similar_name_is_not_the_database_member
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpqyiwrtov\\fixture.db'
======================================================================
ERROR: test_sidecar_or_similar_name_is_not_the_database_member
(__main__.BackupDatabaseMemberTests.test_sidecar_or_similar_name_is_not_the_database_member)
(member='old-jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmp4j8sqngn\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 467, in
test_sidecar_or_similar_name_is_not_the_database_member
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmp4j8sqngn\\fixture.db'
======================================================================
ERROR: test_sidecar_or_similar_name_is_not_the_database_member
(__main__.BackupDatabaseMemberTests.test_sidecar_or_similar_name_is_not_the_database_member)
(member='jobtracker.db.backup')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpdn9dgzlq\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 467, in
test_sidecar_or_similar_name_is_not_the_database_member
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpdn9dgzlq\\fixture.db'
======================================================================
ERROR: test_unsafe_database_member_paths_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_unsafe_database_member_paths_block_the_next_gate) (name='../jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmp1qtpiiuf\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 505, in
test_unsafe_database_member_paths_block_the_next_gate
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmp1qtpiiuf\\fixture.db'
======================================================================
ERROR: test_unsafe_database_member_paths_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_unsafe_database_member_paths_block_the_next_gate) (name='/jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmphovjwsze\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 505, in
test_unsafe_database_member_paths_block_the_next_gate
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmphovjwsze\\fixture.db'
======================================================================
ERROR: test_unsafe_database_member_paths_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_unsafe_database_member_paths_block_the_next_gate)
(name='data/../../jobtracker.db')
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmp00j9gsus\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 505, in
test_unsafe_database_member_paths_block_the_next_gate
result = self.verify_archive(name)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmp00j9gsus\\fixture.db'
======================================================================
ERROR: test_valid_gzip_with_truncated_tar_database_member_is_rejected
(__main__.BackupDatabaseMemberTests.test_valid_gzip_with_truncated_tar_database_member_is_rejected)
----------------------------------------------------------------------
Traceback (most recent call last):
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 633, in _rmtree_unsafe
os.unlink(fullname)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpx9s_a2rf\\fixture.db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 481, in
test_valid_gzip_with_truncated_tar_database_member_is_rejected
result = self.verify_archive('jobtracker.db', truncated_tar=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 424, in verify_archive
with tempfile.TemporaryDirectory() as temporary:
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 950, in __exit__
self.cleanup()
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 954, in cleanup
self._rmtree(self.name, ignore_errors=self._ignore_cleanup_errors)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 934, in _rmtree
_shutil.rmtree(name, onexc=onexc)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 781, in rmtree
return _rmtree_unsafe(path, onexc)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\shutil.py", line 635, in _rmtree_unsafe
onexc(os.unlink, fullname, err)
File "C:\Users\Connor\AppData\Local\Programs\Python\Python312\Lib\tempfile.py", line 909, in onexc
_os.unlink(path)
PermissionError: [WinError 32] The process cannot access the file because it is being used by another process:
'E:\\Storage\\Projects\\Active\\temp
docs\\jobjakt-local-build\\recovery-archive-validation\\.tmp\\tmpx9s_a2rf\\fixture.db'
======================================================================
FAIL: test_empty_or_non_sqlite_database_bytes_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_empty_or_non_sqlite_database_bytes_block_the_next_gate) (contents=b'')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 498, in
test_empty_or_non_sqlite_database_bytes_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_empty_or_non_sqlite_database_bytes_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_empty_or_non_sqlite_database_bytes_block_the_next_gate) (contents=b'unrelated
application bytes')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 498, in
test_empty_or_non_sqlite_database_bytes_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
======================================================================
FAIL: test_empty_or_non_sqlite_database_bytes_block_the_next_gate
(__main__.BackupDatabaseMemberTests.test_empty_or_non_sqlite_database_bytes_block_the_next_gate) (contents=b'SQLite
format 3X\x00')
----------------------------------------------------------------------
Traceback (most recent call last):
File "E:\Storage\Projects\Active\temp
docs\jobjakt-local-build\recovery-archive-validation\scripts\test-deploy-backup.py", line 498, in
test_empty_or_non_sqlite_database_bytes_block_the_next_gate
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
AssertionError: 0 == 0 : FORMAT VERIFIED
NEXT DEPLOY GATE WOULD START
----------------------------------------------------------------------
Ran 6 tests in 3.889s
FAILED (failures=3, errors=14)
+172 -19
View File
@@ -5,6 +5,7 @@ import gzip
from pathlib import Path
import shlex
import shutil
import sqlite3
import subprocess
import sys
import tarfile
@@ -15,6 +16,12 @@ ROOT = Path(__file__).resolve().parents[1]
BASH = 'C:/Program Files/Git/bin/bash.exe' if os.name == 'nt' else shutil.which('bash')
def retention_and_upload_gate(source):
start = source.index('if ! backup_recovery_volumes; then')
after_recovery = source.index('\nfi\n', start) + 4
return source[after_recovery:source.index('\nif [ "${DEPLOY_BACKUP_ONLY:-false}"', after_recovery)]
class BackupConnectionTests(unittest.TestCase):
def backup(self, connection):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
@@ -91,30 +98,35 @@ class BackupConnectionTests(unittest.TestCase):
class BackupUploadTests(unittest.TestCase):
def upload(self, fail_operation=''):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
start = source.index('upload_current_backup_set() {')
function = source[start:source.index('\n}\n', start) + 3]
gate_start = source.index('if ! apply_backup_retention || ! upload_current_backup_set; then')
gate = source[gate_start:source.index('\nfi\n', gate_start) + 4]
functions = []
for name in ['apply_backup_retention', 'upload_current_backup_set']:
start = source.index(name + '() {')
functions.append(source[start:source.index('\n}\n', start) + 3])
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
for filename in ['jobtracker-db-synthetic.sql.gz', 'jobtracker-files-synthetic.tar.gz',
'jobtracker-db-older.sql.gz']:
(folder / filename).touch()
(folder / 'monthly' / '2099-01').mkdir(parents=True)
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
'BACKUP_STAMP=synthetic',
'BACKUP_RCLONE_REMOTE=fixture:encrypted/',
'FAIL_OPERATION=' + shlex.quote(fail_operation),
'date() { echo 2099-01; }',
'apply_backup_retention() { return 0; }',
'date() {',
' if [ "$FAIL_OPERATION" = clock ]; then return 2; fi;',
' if [ "$FAIL_OPERATION" = rollover ] && [ -f "$BACKUP_DIR/tick" ]; then echo 2099-02;',
' else touch "$BACKUP_DIR/tick"; echo 2099-01; fi;',
'}',
'find() { return 0; }',
'rclone() {',
' printf "UPLOAD %s %s %s\\n" "$1" "$3" "$4";',
' if [ "$1" = "$FAIL_OPERATION" ]; then return 23; fi;',
' if [ "$FAIL_OPERATION" = missing-monthly ] && [ -d "$BACKUP_DIR/monthly/2099-01" ]; then',
' mv "$BACKUP_DIR/monthly/2099-01" "$BACKUP_DIR/unavailable-snapshot"; fi;',
'}',
function,
gate,
*functions,
retention_and_upload_gate(source),
'echo "DEPLOY WOULD CONTINUE"',
])
result = subprocess.run([BASH, '--noprofile', '--norc'], input=script,
@@ -136,6 +148,25 @@ class BackupUploadTests(unittest.TestCase):
self.assertIn('DEPLOY WOULD CONTINUE', result.stdout)
self.assertIn('Current recovery set copied to configured off-host remote.', result.stdout)
def test_month_rollover_uploads_the_retained_snapshot(self):
result, calls = self.upload('rollover')
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertIn('UPLOAD copy fixture:encrypted/monthly/2099-01 --immutable', calls)
def test_clock_failure_blocks_retention_and_upload(self):
result, calls = self.upload('clock')
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertEqual([], calls)
self.assertNotIn('Monthly recovery set retained', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
def test_missing_monthly_snapshot_stops_the_gate(self):
result, calls = self.upload('missing-monthly')
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertEqual(2, len(calls))
self.assertNotIn('Current recovery set copied', result.stdout)
self.assertNotIn('DEPLOY WOULD CONTINUE', result.stdout)
def test_daily_copy_failure_stops_before_remaining_files_and_monthly_copy(self):
result, calls = self.upload('copyto')
self.assertEqual(1, result.returncode, result.stdout + result.stderr)
@@ -161,8 +192,7 @@ class BackupRetentionTests(unittest.TestCase):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
start = source.index('apply_backup_retention() {')
function = source[start:source.index('\n}\n', start) + 3]
gate_start = source.index('if ! apply_backup_retention || ! upload_current_backup_set; then')
gate = source[gate_start:source.index('\nfi\n', gate_start) + 4]
gate = retention_and_upload_gate(source)
fixtures = {'jobtracker-db-synthetic.sql.gz': b'database fixture',
'jobtracker-files-synthetic.tar.gz': b'files fixture'}
with tempfile.TemporaryDirectory() as temporary:
@@ -310,11 +340,19 @@ class BackupPermissionTests(unittest.TestCase):
with tempfile.TemporaryDirectory(prefix='backup-permission-') as directory:
target = Path(directory) / 'synthetic-backup.gz'
payload = b'synthetic permission fixture'
if kind == 'recovery':
output = io.BytesIO()
with tarfile.open(fileobj=output, mode='w:gz') as bundle:
member = tarfile.TarInfo('jobtracker.db')
member.size = 16
bundle.addfile(member, io.BytesIO(b'SQLite format 3\x00'))
payload = output.getvalue()
target.write_bytes(payload)
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote(Path(directory).as_posix()),
'TARGET=' + shlex.quote(target.as_posix()),
'python3() { ' + shlex.quote(Path(sys.executable).as_posix()) + ' "$@"; }',
'chmod() { echo "HOST PERMISSIONS"; return ' + str(host_status) + '; }',
'docker() { if [[ "${@: -1}" == *chmod* ]]; then echo "FALLBACK PERMISSIONS"; return ' + str(fallback_status) + '; fi; return 0; }',
'du() { printf "1K\\tsynthetic\\n"; }',
@@ -360,8 +398,92 @@ class BackupPermissionTests(unittest.TestCase):
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
class BackupSqlTrailerTests(unittest.TestCase):
def verify_dump(self, text):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
functions = []
for name in ['verify_backup_file_shape', 'verify_sql_backup']:
start = source.index(name + '() {')
functions.append(source[start:source.index('\n}\n', start) + 3])
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
target = folder / 'synthetic.sql.gz'
with gzip.open(target, 'wb') as bundle:
bundle.write(text.encode('utf-8'))
script = '\n'.join([
'set -euo pipefail',
'report_backup() { echo "FORMAT VERIFIED"; }',
'rm() { echo "INVALID ARTIFACT REJECTION REQUESTED"; }',
*functions,
'if ! verify_sql_backup ' + shlex.quote(target.as_posix()) + '; then exit 1; fi',
'echo "NEXT DEPLOY GATE WOULD START"',
])
return subprocess.run([BASH, '--noprofile', '--norc'], input=script,
text=True, encoding='utf-8', capture_output=True, cwd=ROOT,
timeout=10, env={key: value for key, value in os.environ.items()
if key.upper() in {'PATH', 'SYSTEMROOT', 'WINDIR', 'TEMP', 'TMP'}})
def test_completion_phrase_in_sql_value_is_not_a_dump_trailer(self):
result = self.verify_dump("CREATE TABLE synthetic (value text);\nINSERT INTO synthetic VALUES ('Dump completed');\n")
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_completion_comment_before_more_sql_is_not_a_final_trailer(self):
result = self.verify_dump("CREATE TABLE synthetic (value text);\n-- Dump completed on synthetic-date\nINSERT INTO synthetic VALUES ('more work');\n")
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_final_completion_comment_is_accepted_with_supported_line_endings(self):
for trailer, ending in [('-- Dump completed on synthetic-date', '\n'),
('-- Dump completed', '\n'),
('-- Dump completed on synthetic-date', '\r\n')]:
with self.subTest(trailer=trailer, ending=repr(ending)):
result = self.verify_dump(ending.join(['CREATE TABLE synthetic (id int);', trailer, '', ' ', '']))
self.assertEqual(0, result.returncode, result.stdout + result.stderr)
self.assertIn('FORMAT VERIFIED', result.stdout)
self.assertIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_missing_completion_comment_remains_rejected(self):
result = self.verify_dump('CREATE TABLE synthetic (id int);\n')
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
class BackupDatabaseMemberTests(unittest.TestCase):
def verify_archive(self, member_name, truncated_tar=False):
def test_host_tar_producer_with_database_and_sidecar(self):
producer = shutil.which('tar')
self.assertIsNotNone(producer, 'The host tar command is required for producer qualification.')
with tempfile.TemporaryDirectory() as temporary:
folder = Path(temporary)
volume = folder / 'data'
volume.mkdir()
database = volume / 'jobtracker.db'
connection = sqlite3.connect(database)
try:
with connection:
connection.execute('CREATE TABLE synthetic (id INTEGER PRIMARY KEY)')
connection.execute('INSERT INTO synthetic VALUES (1)')
finally:
connection.close()
(volume / 'jobtracker.db-wal').write_bytes(b'fixture sidecar name only')
for case in ('database-and-sidecar', 'sidecar-only'):
with self.subTest(case=case):
if case == 'sidecar-only':
database.unlink()
archive = case + '.tar.gz'
subprocess.run([producer, 'czf', archive, '-C', 'data', '.'],
cwd=folder, check=True, capture_output=True, timeout=15)
result = subprocess.run([sys.executable, ROOT / 'scripts/validate-sqlite-backup.py', archive],
cwd=folder, capture_output=True, timeout=15)
self.assertEqual(0 if database.exists() else 1, result.returncode, result.stderr)
if database.exists():
with tarfile.open(folder / archive, 'r:gz') as bundle:
with bundle.extractfile('./jobtracker.db') as archived:
self.assertEqual(database.read_bytes(), archived.read())
def verify_archive(self, member_name, truncated_tar=False, member_type=tarfile.REGTYPE, contents=None):
source = (ROOT / 'deploy/deploy.sh').read_text(encoding='utf-8')
functions = []
for name in ['verify_backup_file_shape', 'verify_volume_backup']:
@@ -371,23 +493,30 @@ class BackupDatabaseMemberTests(unittest.TestCase):
folder = Path(temporary)
archive = folder / 'synthetic.tar.gz'
member = tarfile.TarInfo(member_name)
member.type = member_type
if member_type in (tarfile.SYMTYPE, tarfile.LNKTYPE):
member.linkname = 'missing-database'
if contents is None:
database = folder / 'fixture.db'
with sqlite3.connect(database) as connection:
connection.execute('CREATE TABLE synthetic (id INTEGER PRIMARY KEY, value TEXT)')
connection.execute("INSERT INTO synthetic(value) VALUES ('fixture only')")
connection.close()
contents = database.read_bytes()
if truncated_tar:
member.size = 4096
with gzip.open(archive, 'wb') as bundle:
bundle.write(member.tobuf())
else:
member.size = len(b'synthetic bytes')
member.size = len(contents) if member_type == tarfile.REGTYPE else 0
with tarfile.open(archive, 'w:gz') as bundle:
bundle.addfile(member, io.BytesIO(b'synthetic bytes'))
bundle.addfile(member, io.BytesIO(contents) if member_type == tarfile.REGTYPE else None)
script = '\n'.join([
'set -euo pipefail',
'BACKUP_DIR=' + shlex.quote(folder.as_posix()),
'python3() { ' + shlex.quote(Path(sys.executable).as_posix()) + ' "$@"; }',
'report_backup() { echo "FORMAT VERIFIED"; }',
# Replace container-only paths; run its actual tar and grep commands.
r'docker() { local argument script="${!#}"; '
r'for argument in "$@"; do case "$argument" in ARCHIVE_NAME=*) export "$argument";; esac; done; '
r'script="${script//\/backup\//./}"; script="${script//\/tmp\/backup-members/backup-members}"; '
r'(cd "$BACKUP_DIR" && sh -c "$script"); }',
'docker() { echo "Unexpected Docker validation call" >&2; return 99; }',
*functions,
'if ! verify_volume_backup "$BACKUP_DIR/synthetic.tar.gz"; then exit 1; fi',
'echo "NEXT DEPLOY GATE WOULD START"',
@@ -419,6 +548,30 @@ class BackupDatabaseMemberTests(unittest.TestCase):
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_links_and_special_members_do_not_substitute_for_database_bytes(self):
for kind in [tarfile.SYMTYPE, tarfile.LNKTYPE, tarfile.FIFOTYPE, tarfile.CHRTYPE]:
with self.subTest(kind=kind):
result = self.verify_archive('jobtracker.db', member_type=kind)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_empty_or_non_sqlite_database_bytes_block_the_next_gate(self):
for contents in [b'', b'unrelated application bytes', b'SQLite format 3X\x00']:
with self.subTest(contents=contents):
result = self.verify_archive('jobtracker.db', contents=contents)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
def test_unsafe_database_member_paths_block_the_next_gate(self):
for name in ['../jobtracker.db', '/jobtracker.db', 'data/../../jobtracker.db']:
with self.subTest(name=name):
result = self.verify_archive(name)
self.assertNotEqual(0, result.returncode, result.stdout + result.stderr)
self.assertNotIn('FORMAT VERIFIED', result.stdout)
self.assertNotIn('NEXT DEPLOY GATE WOULD START', result.stdout)
if __name__ == '__main__':
unittest.main()
+37
View File
@@ -0,0 +1,37 @@
"""Check database member identity/type/header without extracting an archive.
This is format qualification, not SQLite integrity, a restore or recovery-set proof.
"""
from pathlib import PurePosixPath
import sys
import tarfile
def validate(archive):
found = False
with tarfile.open(archive, mode='r|gz') as bundle:
for member in bundle:
path = PurePosixPath(member.name)
if path.name != 'jobtracker.db':
continue
if path.is_absolute() or '..' in path.parts or '\\' in member.name:
raise ValueError('SQLite backup has an unsafe database member path.')
if not member.isfile() or member.size < 16:
raise ValueError('SQLite backup database must be a nonempty regular file.')
with bundle.extractfile(member) as data:
if data.read(16) != b'SQLite format 3\x00':
raise ValueError('SQLite backup database header is invalid.')
found = True
if not found:
raise ValueError('SQLite backup contains no database member.')
if __name__ == '__main__':
try:
if len(sys.argv) != 2:
raise ValueError('Provide one SQLite backup archive.')
validate(sys.argv[1])
except (ValueError, OSError, EOFError, tarfile.TarError):
# Archive names, member names and database contents may contain private data.
print('SQLite backup member validation failed.', file=sys.stderr)
sys.exit(1)