fix(cv): isolate document parsing
Run untrusted document decoders in a secret-free, resource-bounded child process and terminate its process tree on deadline. Harden the production container and enforce parser and lint gates in CI.
This commit is contained in:
@@ -64,6 +64,11 @@ EXTERNAL_AI_ALLOWED_TASKS=cv-normalize,cv-classify,cv-rewrite
|
|||||||
EXTERNAL_AI_MAX_PROMPT_CHARS=24000
|
EXTERNAL_AI_MAX_PROMPT_CHARS=24000
|
||||||
LOCAL_AI_CIRCUIT_FAILURE_THRESHOLD=3
|
LOCAL_AI_CIRCUIT_FAILURE_THRESHOLD=3
|
||||||
LOCAL_AI_CIRCUIT_OPEN_SECONDS=30
|
LOCAL_AI_CIRCUIT_OPEN_SECONDS=30
|
||||||
|
# Isolated CV document decoder budgets. Increase only after benign-file measurement.
|
||||||
|
PARSER_TIMEOUT_SECONDS=25
|
||||||
|
PARSER_CPU_SECONDS=20
|
||||||
|
PARSER_MEMORY_MIB=512
|
||||||
|
PARSER_STALE_WORK_SECONDS=3600
|
||||||
GEMINI_API_KEY=
|
GEMINI_API_KEY=
|
||||||
GEMINI_MODEL=gemini-2.0-flash
|
GEMINI_MODEL=gemini-2.0-flash
|
||||||
GROQ_API_KEY=
|
GROQ_API_KEY=
|
||||||
|
|||||||
@@ -60,6 +60,18 @@ jobs:
|
|||||||
# without contacting Ollama, pulling a model, or requiring package installation.
|
# without contacting Ollama, pulling a model, or requiring package installation.
|
||||||
run: python3 scripts/test-ollama-evaluation.py
|
run: python3 scripts/test-ollama-evaluation.py
|
||||||
|
|
||||||
|
- name: Test document parser boundary
|
||||||
|
working-directory: tools/summarizer
|
||||||
|
run: |
|
||||||
|
python3 -m venv .venv-ci
|
||||||
|
. .venv-ci/bin/activate
|
||||||
|
python -m pip install --upgrade pip
|
||||||
|
python -m pip install --require-hashes \
|
||||||
|
--extra-index-url https://download.pytorch.org/whl/cpu \
|
||||||
|
-r requirements-linux.lock
|
||||||
|
python -m pip install pytest==8.3.5 httpx2==2.12.0
|
||||||
|
AI_SERVICE_SKIP_MODEL_LOAD=1 python -m pytest -q
|
||||||
|
|
||||||
- name: Restore backend
|
- name: Restore backend
|
||||||
# The runner/proxy has occasionally supplied checksum-invalid NuGet cache entries (NU3008).
|
# The runner/proxy has occasionally supplied checksum-invalid NuGet cache entries (NU3008).
|
||||||
# Retry from clean HTTP/global caches; signature verification remains enabled.
|
# Retry from clean HTTP/global caches; signature verification remains enabled.
|
||||||
@@ -144,6 +156,10 @@ jobs:
|
|||||||
working-directory: job-tracker-ui
|
working-directory: job-tracker-ui
|
||||||
run: npm audit --audit-level=high
|
run: npm audit --audit-level=high
|
||||||
|
|
||||||
|
- name: Lint frontend
|
||||||
|
working-directory: job-tracker-ui
|
||||||
|
run: npm run lint
|
||||||
|
|
||||||
- name: Test frontend
|
- name: Test frontend
|
||||||
working-directory: job-tracker-ui
|
working-directory: job-tracker-ui
|
||||||
# Run the WHOLE suite. Never whitelist test files here again: the previous
|
# Run the WHOLE suite. Never whitelist test files here again: the previous
|
||||||
|
|||||||
@@ -170,6 +170,10 @@ services:
|
|||||||
- EXTERNAL_AI_MAX_PROMPT_CHARS=${EXTERNAL_AI_MAX_PROMPT_CHARS:-24000}
|
- EXTERNAL_AI_MAX_PROMPT_CHARS=${EXTERNAL_AI_MAX_PROMPT_CHARS:-24000}
|
||||||
- LOCAL_AI_CIRCUIT_FAILURE_THRESHOLD=${LOCAL_AI_CIRCUIT_FAILURE_THRESHOLD:-3}
|
- LOCAL_AI_CIRCUIT_FAILURE_THRESHOLD=${LOCAL_AI_CIRCUIT_FAILURE_THRESHOLD:-3}
|
||||||
- LOCAL_AI_CIRCUIT_OPEN_SECONDS=${LOCAL_AI_CIRCUIT_OPEN_SECONDS:-30}
|
- LOCAL_AI_CIRCUIT_OPEN_SECONDS=${LOCAL_AI_CIRCUIT_OPEN_SECONDS:-30}
|
||||||
|
- PARSER_TIMEOUT_SECONDS=${PARSER_TIMEOUT_SECONDS:-25}
|
||||||
|
- PARSER_CPU_SECONDS=${PARSER_CPU_SECONDS:-20}
|
||||||
|
- PARSER_MEMORY_MIB=${PARSER_MEMORY_MIB:-512}
|
||||||
|
- PARSER_STALE_WORK_SECONDS=${PARSER_STALE_WORK_SECONDS:-3600}
|
||||||
- GEMINI_API_KEY=${GEMINI_API_KEY:-}
|
- GEMINI_API_KEY=${GEMINI_API_KEY:-}
|
||||||
- GEMINI_MODEL=${GEMINI_MODEL:-gemini-2.0-flash}
|
- GEMINI_MODEL=${GEMINI_MODEL:-gemini-2.0-flash}
|
||||||
- GROQ_API_KEY=${GROQ_API_KEY:-}
|
- GROQ_API_KEY=${GROQ_API_KEY:-}
|
||||||
@@ -184,6 +188,18 @@ services:
|
|||||||
# re-adding a `ports:` here.
|
# re-adding a `ports:` here.
|
||||||
expose:
|
expose:
|
||||||
- "8001"
|
- "8001"
|
||||||
|
read_only: true
|
||||||
|
cap_drop:
|
||||||
|
- ALL
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
pids_limit: 96
|
||||||
|
mem_limit: 2g
|
||||||
|
cpus: 2.0
|
||||||
|
tmpfs:
|
||||||
|
- /tmp:rw,noexec,nosuid,nodev,size=768m,mode=1777
|
||||||
|
volumes:
|
||||||
|
- ai_model_cache:/home/app/.cache/huggingface
|
||||||
# ai_internal ONLY. Not on `default` (which the frontend shares) and not on
|
# ai_internal ONLY. Not on `default` (which the frontend shares) and not on
|
||||||
# `shared_services` (which is `external: true`, so any other compose stack on this host can
|
# `shared_services` (which is `external: true`, so any other compose stack on this host can
|
||||||
# join it and would then be able to reach this service). ai_internal carries exactly two
|
# join it and would then be able to reach this service). ai_internal carries exactly two
|
||||||
@@ -242,6 +258,7 @@ volumes:
|
|||||||
jobtracker_data:
|
jobtracker_data:
|
||||||
jobtracker_deletion_tombstones:
|
jobtracker_deletion_tombstones:
|
||||||
ollama_data:
|
ollama_data:
|
||||||
|
ai_model_cache:
|
||||||
|
|
||||||
networks:
|
networks:
|
||||||
shared_services:
|
shared_services:
|
||||||
|
|||||||
@@ -222,3 +222,4 @@ Output was reduced to filenames and commit counts. The token artifact appears un
|
|||||||
| V-188 | 75-user query-count fixture, 205-message/two-tenant pagination fixture, focused/full backend and frontend, optimized build | Repository root / `job-tracker-ui` | Remove JT-021's confirmed N+1 and silent 200-message ceiling without breaking old clients | PASS — admin list performs two reads independent of 75 users; page 3 returns the final 5 of 205 owned messages and excludes another tenant. Focused backend 9/9, correspondence UI 15/15, backend 683/683, frontend 58 suites/239 tests and build pass. UI page navigation and filter reset are covered; the legacy inbox endpoint remains unchanged | Synthetic SQLite/InMemory/JSDOM only; no provider mailbox, production dataset or p95 load test. Current page linked/inbound chips intentionally describe the visible page | JT-021 repository defects closed; provider/production capacity remains operational evidence |
|
| V-188 | 75-user query-count fixture, 205-message/two-tenant pagination fixture, focused/full backend and frontend, optimized build | Repository root / `job-tracker-ui` | Remove JT-021's confirmed N+1 and silent 200-message ceiling without breaking old clients | PASS — admin list performs two reads independent of 75 users; page 3 returns the final 5 of 205 owned messages and excludes another tenant. Focused backend 9/9, correspondence UI 15/15, backend 683/683, frontend 58 suites/239 tests and build pass. UI page navigation and filter reset are covered; the legacy inbox endpoint remains unchanged | Synthetic SQLite/InMemory/JSDOM only; no provider mailbox, production dataset or p95 load test. Current page linked/inbound chips intentionally describe the visible page | JT-021 repository defects closed; provider/production capacity remains operational evidence |
|
||||||
| V-189 | Fresh Python 3.12 virtual environments; `pytest -q`; `npm run lint`; full frontend Jest; `npm run build`; `npm audit --audit-level=moderate` | Repository root / `tools/summarizer` / `job-tracker-ui` | Restore the local Python test environment and establish a reproducible zero-warning frontend lint gate | PASS — Python 3.12.10; sidecar 26/26; ESLint zero findings; frontend 64 suites/272 tests; optimized build/TypeScript; npm audit zero vulnerabilities | Five existing SWIG deprecation warnings; Jest retains its documented force-exit/open-handle notice. ESLint 9 is intentionally pinned because the Next 16 React plugin is not ESLint 10 compatible | Local Python/frontend quality gates restored; parser dependency remediation proceeds separately under SEC-006 |
|
| V-189 | Fresh Python 3.12 virtual environments; `pytest -q`; `npm run lint`; full frontend Jest; `npm run build`; `npm audit --audit-level=moderate` | Repository root / `tools/summarizer` / `job-tracker-ui` | Restore the local Python test environment and establish a reproducible zero-warning frontend lint gate | PASS — Python 3.12.10; sidecar 26/26; ESLint zero findings; frontend 64 suites/272 tests; optimized build/TypeScript; npm audit zero vulnerabilities | Five existing SWIG deprecation warnings; Jest retains its documented force-exit/open-handle notice. ESLint 9 is intentionally pinned because the Next 16 React plugin is not ESLint 10 compatible | Local Python/frontend quality gates restored; parser dependency remediation proceeds separately under SEC-006 |
|
||||||
| V-190 | Compatible parser dependency resolution; Linux CPU hash lock; clean hash install; `pip-audit`; generated parser boundary tests; focused/full backend and sidecar suites; Docker daemon probe | Repository root / `tools/summarizer` | Remove reachable upload-parser advisories and close unsafe fallback/resource-boundary paths without parsing hostile fixtures | PASS/PARTIAL — FastAPI 0.141.1/Starlette 1.6.0, Pillow 12.3.0, pypdf 6.16.2 and python-multipart 0.0.32 resolve and install from hashes; upload-facing packages audit clear; parser 32/32, focused backend 51/51 and backend 719/719. Signature/container/page/pixel/decompression/output limits pass; backend binary fallback is removed and unexpected failures are sanitized | Docker daemon unavailable, so production image smoke/container assertions did not run. `pip-audit` reports 45 Torch/Transformers model-stack advisories tracked separately under JT-017. One Starlette/httpx test-client warning and five SWIG warnings remain | SEC-006 implemented locally; SEC-007 boundary work in progress, with child-process and container isolation still required |
|
| V-190 | Compatible parser dependency resolution; Linux CPU hash lock; clean hash install; `pip-audit`; generated parser boundary tests; focused/full backend and sidecar suites; Docker daemon probe | Repository root / `tools/summarizer` | Remove reachable upload-parser advisories and close unsafe fallback/resource-boundary paths without parsing hostile fixtures | PASS/PARTIAL — FastAPI 0.141.1/Starlette 1.6.0, Pillow 12.3.0, pypdf 6.16.2 and python-multipart 0.0.32 resolve and install from hashes; upload-facing packages audit clear; parser 32/32, focused backend 51/51 and backend 719/719. Signature/container/page/pixel/decompression/output limits pass; backend binary fallback is removed and unexpected failures are sanitized | Docker daemon unavailable, so production image smoke/container assertions did not run. `pip-audit` reports 45 Torch/Transformers model-stack advisories tracked separately under JT-017. One Starlette/httpx test-client warning and five SWIG warnings remain | SEC-006 implemented locally; SEC-007 boundary work in progress, with child-process and container isolation still required |
|
||||||
|
| V-191 | Real parser child extraction; minimal-environment probe; timeout/process-tree/capacity/stale-cleanup tests; `py_compile`; parser suite; Compose interpolation/control inspection | Repository root / `tools/summarizer` | Ensure untrusted document decode cannot consume the AI web process or inherit provider secrets and receives explicit runtime/container budgets | PASS/PARTIAL — parser 37/37 without warnings; TXT extraction executes in a child; provider/service secrets are absent; timeout kills parent and descendant; capacity recovers; stale cleanup preserves recent/unrelated paths. Compose resolves read-only root, `cap_drop: ALL`, no-new-privileges, 96 PIDs, 2 CPUs, 2 GiB memory, 768 MiB tmpfs and one model-cache volume | Windows proves deadline/process-tree behavior but cannot execute Linux `setrlimit`; Docker Desktop daemon is offline, so image build, non-root identity, inside-container rlimits and benign PDF/DOCX/image sizing remain unverified | SEC-007 repository boundary implemented; Linux/container/browser/production verification remains |
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# SEC-007 parser isolation verification
|
||||||
|
|
||||||
|
## Implemented boundary
|
||||||
|
|
||||||
|
- The FastAPI web process reads at most 5 MiB and performs cheap extension, signature and DOCX-container preflight checks.
|
||||||
|
- TXT, Markdown, PDF, DOCX and image decoding runs in `parser_child.py`, never in the web process.
|
||||||
|
- Only one parser child is admitted at a time; excess work receives a stable busy response instead of building an in-memory parser queue.
|
||||||
|
- The child receives a minimal environment without AI provider keys or the service token.
|
||||||
|
- Linux applies CPU, address-space, output-file, open-file and core-dump limits before importing parser libraries.
|
||||||
|
- The parent enforces a 25-second deadline and terminates the process group/tree, including OCR descendants.
|
||||||
|
- Per-request work uses an owner-private temporary directory. Old `work-*` directories are reconciled without following symlinks or removing unrelated paths.
|
||||||
|
- The backend has no PDF/DOCX/image fallback. Stable messages replace unexpected internal exceptions and paths.
|
||||||
|
|
||||||
|
## Container boundary
|
||||||
|
|
||||||
|
The production Compose definition resolves the AI service with:
|
||||||
|
|
||||||
|
- non-root Dockerfile user;
|
||||||
|
- read-only root filesystem;
|
||||||
|
- all Linux capabilities dropped;
|
||||||
|
- `no-new-privileges`;
|
||||||
|
- 96 PID, 2 CPU and 2 GiB memory limits;
|
||||||
|
- 768 MiB `noexec,nosuid,nodev` `/tmp` tmpfs;
|
||||||
|
- one writable named volume only for the Hugging Face model cache.
|
||||||
|
|
||||||
|
## Local proof
|
||||||
|
|
||||||
|
- Parser tests: 37/37 passed.
|
||||||
|
- Focused backend extraction/operation tests: 51/51 passed.
|
||||||
|
- Full backend: 719/719 passed.
|
||||||
|
- Python compilation: `app.py` and `parser_child.py` passed.
|
||||||
|
- Compose config: valid with all intended controls present.
|
||||||
|
- Timeout regression spawns a harmless sleeping descendant and proves the process tree is gone after the deadline.
|
||||||
|
- Environment regression proves `GEMINI_API_KEY`, `GROQ_API_KEY` and `AI_SERVICE_TOKEN` are absent from the child.
|
||||||
|
|
||||||
|
## Remaining external proof
|
||||||
|
|
||||||
|
Docker Desktop's Linux daemon is offline, so the image could not be built or run. Before activation:
|
||||||
|
|
||||||
|
1. Build the hash-locked image and record its digest.
|
||||||
|
2. Confirm the runtime UID is non-root and the root filesystem rejects writes.
|
||||||
|
3. Inspect the applied CPU/memory/PID/capability/tmpfs settings.
|
||||||
|
4. Exercise benign TXT, Markdown, PDF, DOCX and image extraction within the configured budgets.
|
||||||
|
5. Measure peak parent/child memory and OCR duration; lower or evidence any limit increase.
|
||||||
|
6. Run the authenticated browser upload/review journey with synthetic files.
|
||||||
|
7. Canary in production only after explicit authorization; disable CV import rather than removing the boundary if the budget is insufficient.
|
||||||
@@ -44,7 +44,7 @@ Updated: 2026-08-30
|
|||||||
|
|
||||||
### In progress
|
### In progress
|
||||||
|
|
||||||
- SEC-007 bounded document processing. Upload-facing parser dependencies are fixed and hash-locked; the sidecar now rejects signature mismatches, decompression/page/pixel/output excesses, the backend no longer retries PDF/DOCX/image parsing in-process, and unexpected parser details are sanitized. Child-process deadline/resource isolation and container controls remain.
|
- No repository implementation package is currently in progress. SEC-006/SEC-007 await a running Linux Docker daemon for image/runtime proof; production activation remains separately gated.
|
||||||
|
|
||||||
### Remaining
|
### Remaining
|
||||||
|
|
||||||
@@ -108,17 +108,17 @@ Updated: 2026-08-30
|
|||||||
- Playwright: 8/10 passed on the first complete run; both failures were ambiguous selectors in the newly responsive Career selector, not product failures. Both corrected focused regressions now pass (2/2); final full rerun remains in the end-of-batch gate.
|
- Playwright: 8/10 passed on the first complete run; both failures were ambiguous selectors in the newly responsive Career selector, not product failures. Both corrected focused regressions now pass (2/2); final full rerun remains in the end-of-batch gate.
|
||||||
- Manual desktop browser review: webpack development server rendered the new Career navigation and Overview correctly in dark mode; API-dependent profile status remained unavailable because the backend was not running for that isolated UI review.
|
- Manual desktop browser review: webpack development server rendered the new Career navigation and Overview correctly in dark mode; API-dependent profile status remained unavailable because the backend was not running for that isolated UI review.
|
||||||
|
|
||||||
- **Overall programme status:** Active but externally blocked. Eight packages are locally verified, twenty-six are implemented with verification incomplete, and SEC-007 is in progress. The prioritized admin-only version indicator, every immediate repository/browser item, SEC-009, the PROD-001 read-only inventory, and the PROD-003 safe benchmark harness are complete on the release branch.
|
- **Overall programme status:** Active but externally blocked. Eight packages are locally verified and twenty-seven are implemented with verification incomplete. The prioritized admin-only version indicator, every immediate repository/browser item, SEC-006/SEC-007 repository boundaries, SEC-009, the PROD-001 read-only inventory, and the PROD-003 safe benchmark harness are complete on the feature branch.
|
||||||
- **Current work package:** SEC-007 bounded isolated document processing; SEC-006 is implemented locally and awaits Docker image smoke.
|
- **Current work package:** Reconcile the next safe repository item while SEC-006/SEC-007 await Linux image/runtime proof.
|
||||||
- **Completed work packages:** None are `DONE`; all repository security/AI packages still have applicable browser, provider and/or production gates.
|
- **Completed work packages:** None are `DONE`; all repository security/AI packages still have applicable browser, provider and/or production gates.
|
||||||
- **Locally verified work:** SEC-001, SEC-002, SEC-003, SEC-005A, CORE-001, PROD-002, DEP-001 and VER-001 (`VERIFIED LOCALLY`).
|
- **Locally verified work:** SEC-001, SEC-002, SEC-003, SEC-005A, CORE-001, PROD-002, DEP-001 and VER-001 (`VERIFIED LOCALLY`).
|
||||||
- **Implemented, verification incomplete:** SEC-004, SEC-005B, SEC-006, SEC-008, SEC-009, CORE-002, BG-001, OPS-001A/B/C, POL-001/002, AI-001/002/003/004, UX-001/002/003, QA-001, CAREER-001/002, MAIL-001, JOBS-001/002 and PRODUCT-001 (`IMPLEMENTED — NOT VERIFIED`). Their safe repository/browser scope is implemented; production/native-device/provider/retention gates remain where recorded.
|
- **Implemented, verification incomplete:** SEC-004, SEC-005B, SEC-006, SEC-007, SEC-008, SEC-009, CORE-002, BG-001, OPS-001A/B/C, POL-001/002, AI-001/002/003/004, UX-001/002/003, QA-001, CAREER-001/002, MAIL-001, JOBS-001/002 and PRODUCT-001 (`IMPLEMENTED — NOT VERIFIED`). Their safe repository/browser scope is implemented; production/native-device/provider/retention gates remain where recorded.
|
||||||
- **Production-verified work:** None.
|
- **Production-verified work:** None.
|
||||||
- **Blocked work:** PROD-001/003/004 and REL-001 require network/backup/model/deployment authority and unfinished dependencies. Real provider and live deletion/restore checks remain gated; DEP-001 awaits approved merge/live verification.
|
- **Blocked work:** PROD-001/003/004 and REL-001 require network/backup/model/deployment authority and unfinished dependencies. Real provider and live deletion/restore checks remain gated; DEP-001 awaits approved merge/live verification.
|
||||||
- **Deferred work:** None. Conditional multi-replica coordination, model deletion, realtime operation delivery and unrelated production changes remain outside current packages.
|
- **Deferred work:** None. Conditional multi-replica coordination, model deletion, realtime operation delivery and unrelated production changes remain outside current packages.
|
||||||
- **Immediate order:** all sixteen immediate repository items are complete locally, including the original UI/release queue plus SEC-009 cache/tombstone safety, worker restart clocks, universal AI accounting, email-token/Stripe lifecycle tests, exhaustive Job email selectors, the repaired migration chain, CV/public-edge hardening and measured admin/mail scaling. PROD-001 read-only evidence and the PROD-003 plan-only harness are also complete. The final audit is checking tooling/documentation before declaring only external blockers remain.
|
- **Immediate order:** all sixteen immediate repository items are complete locally, including the original UI/release queue plus SEC-009 cache/tombstone safety, worker restart clocks, universal AI accounting, email-token/Stripe lifecycle tests, exhaustive Job email selectors, the repaired migration chain, CV/public-edge hardening and measured admin/mail scaling. PROD-001 read-only evidence and the PROD-003 plan-only harness are also complete. The final audit is checking tooling/documentation before declaring only external blockers remain.
|
||||||
- **Status counts:** 8 `VERIFIED LOCALLY`; 26 `IMPLEMENTED — NOT VERIFIED`; 1 `IN PROGRESS`; 0 `NOT STARTED`; 4 `BLOCKED`; 0 `DONE`; 0 `DEFERRED`.
|
- **Status counts:** 8 `VERIFIED LOCALLY`; 27 `IMPLEMENTED — NOT VERIFIED`; 0 `IN PROGRESS`; 0 `NOT STARTED`; 4 `BLOCKED`; 0 `DONE`; 0 `DEFERRED`.
|
||||||
- **Test status:** backend 719/719; frontend 64/64 suites and 272/272 tests; ESLint zero findings; AI sidecar 32/32; Ollama benchmark harness 5/5 plus safe dry-run; optimized production build/TypeScript; EF model parity; SQLite/MariaDB migration scripts; blank/idempotent/populated SQLite migration-chain tests; disposable fresh/restarted MariaDB 11.8 application startup; Docker Compose config; safe-failure deployment preflight; and Playwright 9/9. npm audit remains at zero. The parser image build is pending because Docker Desktop's daemon is offline. Jest's slow/open-handle behavior and six Python deprecation warnings remain recorded.
|
- **Test status:** backend 719/719; frontend 64/64 suites and 272/272 tests; ESLint zero findings; AI sidecar 37/37 without warnings; Ollama benchmark harness 5/5 plus safe dry-run; optimized production build/TypeScript; EF model parity; SQLite/MariaDB migration scripts; blank/idempotent/populated SQLite migration-chain tests; disposable fresh/restarted MariaDB 11.8 application startup; Docker Compose config; safe-failure deployment preflight; and Playwright 9/9. npm audit remains at zero. The parser image build is pending because Docker Desktop's daemon is offline. Jest's slow/open-handle behavior remains recorded.
|
||||||
- **Deployment status:** Gitea pull-request run 609 passed the complete CI job in 4m20s. Deploy was intentionally skipped because the workflow deploys only a `push` to `main`; live remains unchanged. No merge/deployment was performed directly, no production migrations were run and the AI operation worker remains disabled by default.
|
- **Deployment status:** Gitea pull-request run 609 passed the complete CI job in 4m20s. Deploy was intentionally skipped because the workflow deploys only a `push` to `main`; live remains unchanged. No merge/deployment was performed directly, no production migrations were run and the AI operation worker remains disabled by default.
|
||||||
- **Production status:** State unchanged. Sanitized read-only SSH inventory was performed; no logs, prompts, private rows/content or secret values were read, and no provider/model call, model pull, service restart, file/config change, backup, restore, migration or deployment occurred. It confirmed all-interface Ollama/frontend listeners and stale database-only backups as rollout blockers.
|
- **Production status:** State unchanged. Sanitized read-only SSH inventory was performed; no logs, prompts, private rows/content or secret values were read, and no provider/model call, model pull, service restart, file/config change, backup, restore, migration or deployment occurred. It confirmed all-interface Ollama/frontend listeners and stale database-only backups as rollout blockers.
|
||||||
- **Known regressions:** None found by automated/local browser checks. Jest still needs `--forceExit` and reports its existing open-handle notice. Email-provider/send tests are fake/local only; real delivery is not claimed. Current MAIL browser evidence is 1280×720 only because the browser surface could not resize or perform native Tab traversal. Interrupted attempts are aged after 15 minutes and notified without retry; the five-minute scan is unmeasured on a large ledger. The direct clean EF-only SQLite defect and synchronous AI accounting gap are closed; migration/reconciler dual ownership remains architectural debt.
|
- **Known regressions:** None found by automated/local browser checks. Jest still needs `--forceExit` and reports its existing open-handle notice. Email-provider/send tests are fake/local only; real delivery is not claimed. Current MAIL browser evidence is 1280×720 only because the browser surface could not resize or perform native Tab traversal. Interrupted attempts are aged after 15 minutes and notified without retry; the five-minute scan is unmeasured on a large ledger. The direct clean EF-only SQLite defect and synchronous AI accounting gap are closed; migration/reconciler dual ownership remains architectural debt.
|
||||||
|
|||||||
@@ -240,11 +240,11 @@ This queue records the highest-value work that can proceed without production cr
|
|||||||
- **Required tests:** generated boundary/corrupt fixtures, harmless sleeping child, cancellation/restart cleanup, outage/no-fallback, container assertions.
|
- **Required tests:** generated boundary/corrupt fixtures, harmless sleeping child, cancellation/restart cleanup, outage/no-fallback, container assertions.
|
||||||
- **Required browser verification:** synthetic CV upload status/failure; authorized private CV local-only only after safeguards.
|
- **Required browser verification:** synthetic CV upload status/failure; authorized private CV local-only only after safeguards.
|
||||||
- **Required production verification:** measured memory/CPU limits and canary synthetic extraction.
|
- **Required production verification:** measured memory/CPU limits and canary synthetic extraction.
|
||||||
- **Status:** `IN PROGRESS`.
|
- **Status:** `IMPLEMENTED — NOT VERIFIED`.
|
||||||
- **Blocker:** production sizing requires access; local Docker container assertions require a running daemon.
|
- **Blocker:** production sizing and canary require access; an actual Linux image build/runtime smoke requires a running Docker daemon.
|
||||||
- **Evidence:** signature/container validation, page/pixel/decompression/output ceilings and stable parser errors have boundary tests; backend binary fallback is removed and raw unexpected failures are sanitized. Focused backend 51/51, full backend 719/719 and parser 32/32 pass.
|
- **Evidence:** `docs/verification/sec-007-parser-isolation.md`; child process/minimal environment/deadline/process-tree termination/stale cleanup tests pass; Compose resolves non-root-image/read-only/capability/PID/CPU/memory/tmpfs controls. Parser 37/37, focused backend 51/51 and full backend 719/719 pass.
|
||||||
- **Commit:** none.
|
- **Commit:** none.
|
||||||
- **Remaining work:** move decode into a deadline-bounded child process/process group, add descendant termination and cleanup tests, bound scheduling/backpressure, then apply and verify non-root/read-only/PID/CPU/memory/tmpfs container controls.
|
- **Remaining work:** build and run the Linux image, prove rlimits and container controls from inside it, measure benign PDF/DOCX/image memory/CPU, perform the synthetic browser upload failure/success journey, then canary only with explicit production authorization.
|
||||||
|
|
||||||
### SEC-008 — Recoverable attachment mutations
|
### SEC-008 — Recoverable attachment mutations
|
||||||
|
|
||||||
|
|||||||
@@ -2,8 +2,11 @@ FROM python:3.12.10-slim-bookworm
|
|||||||
ENV PIP_NO_CACHE_DIR=1 \
|
ENV PIP_NO_CACHE_DIR=1 \
|
||||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||||
PYTHONUNBUFFERED=1 \
|
PYTHONUNBUFFERED=1 \
|
||||||
|
PYTHONDONTWRITEBYTECODE=1 \
|
||||||
TRANSFORMERS_NO_TF=1 \
|
TRANSFORMERS_NO_TF=1 \
|
||||||
HF_HUB_DISABLE_TELEMETRY=1
|
HF_HUB_DISABLE_TELEMETRY=1 \
|
||||||
|
HF_HOME=/home/app/.cache/huggingface \
|
||||||
|
PARSER_TEMP_ROOT=/tmp
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng \
|
&& apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-eng \
|
||||||
@@ -12,5 +15,10 @@ COPY requirements-linux.lock ./
|
|||||||
RUN python -m pip install --upgrade pip setuptools wheel \
|
RUN python -m pip install --upgrade pip setuptools wheel \
|
||||||
&& python -m pip install --require-hashes --extra-index-url https://download.pytorch.org/whl/cpu -r requirements-linux.lock
|
&& python -m pip install --require-hashes --extra-index-url https://download.pytorch.org/whl/cpu -r requirements-linux.lock
|
||||||
COPY . .
|
COPY . .
|
||||||
|
RUN groupadd --system app \
|
||||||
|
&& useradd --system --gid app --home-dir /home/app --create-home app \
|
||||||
|
&& mkdir -p /home/app/.cache/huggingface \
|
||||||
|
&& chown -R app:app /app /home/app
|
||||||
|
USER app
|
||||||
EXPOSE 8001
|
EXPOSE 8001
|
||||||
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8001"]
|
CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8001"]
|
||||||
|
|||||||
@@ -39,6 +39,14 @@ If the host is missing `python3-venv` or `pip`, use the bootstrap script instead
|
|||||||
## Docker
|
## Docker
|
||||||
The Dockerfile installs Tesseract OCR so scanned PDFs and supported images can be processed inside the container.
|
The Dockerfile installs Tesseract OCR so scanned PDFs and supported images can be processed inside the container.
|
||||||
|
|
||||||
|
## Document parser isolation
|
||||||
|
|
||||||
|
`/extract-text` performs only bounded upload and signature checks in the web process. Actual TXT, Markdown, PDF, DOCX and image decoding runs in a single-capacity child process with a minimal environment. Linux children receive CPU, address-space, output-size and file-descriptor limits; every platform enforces a parent deadline and terminates the parser process tree on timeout.
|
||||||
|
|
||||||
|
Production Compose runs the service as a non-root user with a read-only root filesystem, no Linux capabilities, `no-new-privileges`, PID/CPU/memory limits and a bounded `/tmp` tmpfs. The Hugging Face cache is the only named writable volume. Parser work directories are removed after each request and stale `work-*` directories are reconciled on the next extraction.
|
||||||
|
|
||||||
|
Parser limits can be made more conservative with `PARSER_TIMEOUT_SECONDS`, `PARSER_CPU_SECONDS`, `PARSER_MEMORY_MIB` and `PARSER_STALE_WORK_SECONDS`. Increasing them should follow measured benign-CV evidence; disabling import is the safe fallback when the budget is insufficient.
|
||||||
|
|
||||||
## Tests
|
## Tests
|
||||||
|
|
||||||
Run the summarizer unit tests with:
|
Run the summarizer unit tests with:
|
||||||
|
|||||||
+161
-144
@@ -3,18 +3,20 @@ from fastapi.responses import JSONResponse
|
|||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field
|
||||||
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
|
||||||
from cachetools import TTLCache
|
from cachetools import TTLCache
|
||||||
from PIL import Image
|
import asyncio
|
||||||
from pypdf import PdfReader
|
|
||||||
from docx import Document
|
|
||||||
import fitz
|
|
||||||
import hashlib
|
import hashlib
|
||||||
import hmac
|
import hmac
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
import torch
|
import torch
|
||||||
import pytesseract
|
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
import zipfile
|
import zipfile
|
||||||
@@ -94,17 +96,22 @@ MAX_INPUT_CHARS = 20000
|
|||||||
MAX_CONTEXT_CHARS = 2200
|
MAX_CONTEXT_CHARS = 2200
|
||||||
MAX_EXTRACT_FILE_BYTES = 5 * 1024 * 1024
|
MAX_EXTRACT_FILE_BYTES = 5 * 1024 * 1024
|
||||||
MAX_EXTRACT_CHARS = 200_000
|
MAX_EXTRACT_CHARS = 200_000
|
||||||
MAX_PDF_PAGES = 40
|
|
||||||
MAX_DOCX_ENTRIES = 256
|
MAX_DOCX_ENTRIES = 256
|
||||||
MAX_DOCX_UNCOMPRESSED_BYTES = 32 * 1024 * 1024
|
MAX_DOCX_UNCOMPRESSED_BYTES = 32 * 1024 * 1024
|
||||||
MAX_DOCX_ENTRY_BYTES = 8 * 1024 * 1024
|
MAX_DOCX_ENTRY_BYTES = 8 * 1024 * 1024
|
||||||
MAX_DOCX_COMPRESSION_RATIO = 100
|
MAX_DOCX_COMPRESSION_RATIO = 100
|
||||||
MAX_IMAGE_DIMENSION = 12_000
|
|
||||||
MAX_IMAGE_PIXELS = 40_000_000
|
|
||||||
MAX_PDF_PAGE_PIXELS = 12_000_000
|
|
||||||
MAX_PDF_OCR_PIXELS = 120_000_000
|
|
||||||
OCR_LANGUAGES = "eng"
|
OCR_LANGUAGES = "eng"
|
||||||
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
|
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
|
||||||
|
PARSER_CHILD_PATH = Path(__file__).with_name("parser_child.py")
|
||||||
|
PARSER_TIMEOUT_SECONDS = max(5, min(int(os.getenv("PARSER_TIMEOUT_SECONDS", "25")), 120))
|
||||||
|
PARSER_CPU_SECONDS = max(2, min(int(os.getenv("PARSER_CPU_SECONDS", "20")), PARSER_TIMEOUT_SECONDS))
|
||||||
|
PARSER_MEMORY_BYTES = max(256, min(int(os.getenv("PARSER_MEMORY_MIB", "512")), 2048)) * 1024 * 1024
|
||||||
|
PARSER_MAX_RESULT_BYTES = 1024 * 1024
|
||||||
|
_parser_capacity = threading.BoundedSemaphore(value=1)
|
||||||
|
PARSER_TEMP_ROOT = Path(os.getenv("PARSER_TEMP_ROOT", tempfile.gettempdir()), "jobtracker-cv-parser").resolve()
|
||||||
|
PARSER_STALE_WORK_SECONDS = max(300, min(int(os.getenv("PARSER_STALE_WORK_SECONDS", "3600")), 86_400))
|
||||||
|
_parser_temp_lock = threading.Lock()
|
||||||
|
_parser_temp_ready = False
|
||||||
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/")
|
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/")
|
||||||
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "")
|
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "")
|
||||||
|
|
||||||
@@ -280,6 +287,11 @@ async def health():
|
|||||||
"gpu_name": GPU_NAME,
|
"gpu_name": GPU_NAME,
|
||||||
"ocr_available": True,
|
"ocr_available": True,
|
||||||
"ocr_languages": OCR_LANGUAGES,
|
"ocr_languages": OCR_LANGUAGES,
|
||||||
|
"parser_isolated": True,
|
||||||
|
"parser_concurrency": 1,
|
||||||
|
"parser_timeout_seconds": PARSER_TIMEOUT_SECONDS,
|
||||||
|
"parser_cpu_seconds": PARSER_CPU_SECONDS,
|
||||||
|
"parser_memory_mib": PARSER_MEMORY_BYTES // (1024 * 1024),
|
||||||
"model_loaded": MODEL_LOADED,
|
"model_loaded": MODEL_LOADED,
|
||||||
"model_disabled": MODEL_DISABLED,
|
"model_disabled": MODEL_DISABLED,
|
||||||
"summarize_available": MODEL_LOADED and not MODEL_DISABLED,
|
"summarize_available": MODEL_LOADED and not MODEL_DISABLED,
|
||||||
@@ -1055,47 +1067,6 @@ async def purge_cache():
|
|||||||
return {"cleared": cleared}
|
return {"cleared": cleared}
|
||||||
|
|
||||||
|
|
||||||
def _normalize_text(value: str) -> str:
|
|
||||||
"""Normalize extraction noise without destroying section, bullet, or table boundaries."""
|
|
||||||
value = value.replace("\x00", " ").replace("\r\n", "\n").replace("\r", "\n")
|
|
||||||
lines = []
|
|
||||||
blank = False
|
|
||||||
for raw_line in value.split("\n"):
|
|
||||||
line = re.sub(r"[\t\f\v]+", " ", raw_line).strip()
|
|
||||||
if not line:
|
|
||||||
if lines and not blank:
|
|
||||||
lines.append("")
|
|
||||||
blank = True
|
|
||||||
continue
|
|
||||||
lines.append(line)
|
|
||||||
blank = False
|
|
||||||
return "\n".join(lines).strip()
|
|
||||||
|
|
||||||
|
|
||||||
def _ocr_image(image: Image.Image) -> str:
|
|
||||||
if image.mode not in ("RGB", "L"):
|
|
||||||
image = image.convert("RGB")
|
|
||||||
text = pytesseract.image_to_string(image, lang=OCR_LANGUAGES)
|
|
||||||
return _normalize_text(text)
|
|
||||||
|
|
||||||
|
|
||||||
def _check_extracted_size(text: str) -> str:
|
|
||||||
if len(text) > MAX_EXTRACT_CHARS:
|
|
||||||
raise HTTPException(status_code=422, detail="The document contains too much extracted text.")
|
|
||||||
return text
|
|
||||||
|
|
||||||
|
|
||||||
def _check_image_size(image: Image.Image, *, pixel_limit: int | None = None) -> None:
|
|
||||||
pixel_limit = MAX_IMAGE_PIXELS if pixel_limit is None else pixel_limit
|
|
||||||
width, height = image.size
|
|
||||||
if width <= 0 or height <= 0 or width > MAX_IMAGE_DIMENSION or height > MAX_IMAGE_DIMENSION:
|
|
||||||
raise HTTPException(status_code=422, detail="The document image dimensions are not supported.")
|
|
||||||
if width * height > pixel_limit:
|
|
||||||
raise HTTPException(status_code=422, detail="The document image contains too many pixels.")
|
|
||||||
if getattr(image, "n_frames", 1) != 1:
|
|
||||||
raise HTTPException(status_code=422, detail="Multi-frame document images are not supported.")
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_docx_container(data: bytes) -> None:
|
def _validate_docx_container(data: bytes) -> None:
|
||||||
try:
|
try:
|
||||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||||
@@ -1137,76 +1108,144 @@ def _validate_file_signature(extension: str, data: bytes) -> None:
|
|||||||
raise HTTPException(status_code=400, detail="The uploaded text file contains binary data.")
|
raise HTTPException(status_code=400, detail="The uploaded text file contains binary data.")
|
||||||
|
|
||||||
|
|
||||||
def _extract_pdf_text(data: bytes) -> tuple[str, bool, int]:
|
_PARSER_ERRORS = {
|
||||||
page_count = 0
|
"unsupported_type": (400, "This file type is not supported for AI extraction."),
|
||||||
extracted_pages = []
|
"text_encoding": (400, "The uploaded text file is not valid UTF-8."),
|
||||||
|
"docx_signature": (400, "The uploaded file does not match its DOCX extension."),
|
||||||
|
"docx_entry_limit": (422, "The DOCX contains too many files."),
|
||||||
|
"docx_entry_size": (422, "The DOCX contains an oversized file."),
|
||||||
|
"docx_ratio": (422, "The DOCX compression ratio is not supported."),
|
||||||
|
"docx_expanded_size": (422, "The DOCX expands beyond the supported size."),
|
||||||
|
"pdf_page_limit": (422, "The PDF contains too many pages."),
|
||||||
|
"pdf_ocr_limit": (422, "The PDF requires too much OCR processing."),
|
||||||
|
"image_dimensions": (422, "The document image dimensions are not supported."),
|
||||||
|
"image_pixel_limit": (422, "The document image contains too many pixels."),
|
||||||
|
"image_frames": (422, "Multi-frame document images are not supported."),
|
||||||
|
"extracted_text_limit": (422, "The document contains too much extracted text."),
|
||||||
|
"no_text": (422, "AI extraction did not find readable text in the uploaded file."),
|
||||||
|
"parser_failed": (422, "Document extraction failed."),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _parser_environment(temp_root: str) -> dict[str, str]:
|
||||||
|
allowed = ("PATH", "LANG", "LC_ALL", "TESSDATA_PREFIX", "SYSTEMROOT", "WINDIR")
|
||||||
|
environment = {key: os.environ[key] for key in allowed if os.environ.get(key)}
|
||||||
|
environment.update({
|
||||||
|
"PYTHONUTF8": "1",
|
||||||
|
"PYTHONDONTWRITEBYTECODE": "1",
|
||||||
|
"OMP_NUM_THREADS": "1",
|
||||||
|
"OPENBLAS_NUM_THREADS": "1",
|
||||||
|
"TMPDIR": temp_root,
|
||||||
|
"TEMP": temp_root,
|
||||||
|
"TMP": temp_root,
|
||||||
|
})
|
||||||
|
return environment
|
||||||
|
|
||||||
|
|
||||||
|
def _prepare_parser_temp_root() -> Path:
|
||||||
|
global _parser_temp_ready
|
||||||
|
with _parser_temp_lock:
|
||||||
|
PARSER_TEMP_ROOT.mkdir(mode=0o700, parents=True, exist_ok=True)
|
||||||
|
if _parser_temp_ready:
|
||||||
|
return PARSER_TEMP_ROOT
|
||||||
|
cutoff = time.time() - PARSER_STALE_WORK_SECONDS
|
||||||
|
for candidate in PARSER_TEMP_ROOT.glob("work-*"):
|
||||||
try:
|
try:
|
||||||
reader = PdfReader(io.BytesIO(data))
|
if candidate.is_symlink() or not candidate.is_dir():
|
||||||
page_count = len(reader.pages)
|
continue
|
||||||
if page_count > MAX_PDF_PAGES:
|
resolved = candidate.resolve()
|
||||||
raise HTTPException(status_code=422, detail="The PDF contains too many pages.")
|
if resolved.parent != PARSER_TEMP_ROOT or candidate.stat().st_mtime > cutoff:
|
||||||
for page in reader.pages:
|
continue
|
||||||
|
shutil.rmtree(resolved)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
_parser_temp_ready = True
|
||||||
|
return PARSER_TEMP_ROOT
|
||||||
|
|
||||||
|
|
||||||
|
def _terminate_parser_process(process: subprocess.Popen) -> None:
|
||||||
|
if process.poll() is not None:
|
||||||
|
return
|
||||||
|
if os.name == "posix":
|
||||||
try:
|
try:
|
||||||
extracted_pages.append(page.extract_text(extraction_mode="layout") or "")
|
os.killpg(process.pid, signal.SIGKILL)
|
||||||
except TypeError:
|
except ProcessLookupError:
|
||||||
extracted_pages.append(page.extract_text() or "")
|
pass
|
||||||
except HTTPException:
|
else:
|
||||||
raise
|
|
||||||
except Exception:
|
|
||||||
extracted_pages = []
|
|
||||||
|
|
||||||
text = _check_extracted_size(_normalize_text("\n".join(extracted_pages)))
|
|
||||||
if len(text) >= 80:
|
|
||||||
return text, False, page_count
|
|
||||||
|
|
||||||
doc = fitz.open(stream=data, filetype="pdf")
|
|
||||||
page_count = doc.page_count
|
|
||||||
if page_count > MAX_PDF_PAGES:
|
|
||||||
doc.close()
|
|
||||||
raise HTTPException(status_code=422, detail="The PDF contains too many pages.")
|
|
||||||
ocr_pages = []
|
|
||||||
total_pixels = 0
|
|
||||||
for page in doc:
|
|
||||||
pix = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
|
|
||||||
page_pixels = pix.width * pix.height
|
|
||||||
total_pixels += page_pixels
|
|
||||||
if page_pixels > MAX_PDF_PAGE_PIXELS or total_pixels > MAX_PDF_OCR_PIXELS:
|
|
||||||
doc.close()
|
|
||||||
raise HTTPException(status_code=422, detail="The PDF requires too much OCR processing.")
|
|
||||||
image = Image.open(io.BytesIO(pix.tobytes("png")))
|
|
||||||
_check_image_size(image, pixel_limit=MAX_PDF_PAGE_PIXELS)
|
|
||||||
ocr_pages.append(_ocr_image(image))
|
|
||||||
_check_extracted_size("\n".join(ocr_pages))
|
|
||||||
doc.close()
|
|
||||||
return _check_extracted_size(_normalize_text("\n".join(ocr_pages))), True, page_count
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_docx_text(data: bytes) -> str:
|
|
||||||
_validate_docx_container(data)
|
|
||||||
document = Document(io.BytesIO(data))
|
|
||||||
parts = []
|
|
||||||
blocks = document.iter_inner_content() if hasattr(document, "iter_inner_content") else document.paragraphs
|
|
||||||
for block in blocks:
|
|
||||||
if hasattr(block, "rows"):
|
|
||||||
for row in block.rows:
|
|
||||||
cells = [cell.text.strip() for cell in row.cells if cell.text and cell.text.strip()]
|
|
||||||
if cells:
|
|
||||||
parts.append(" | ".join(cells))
|
|
||||||
elif getattr(block, "text", "").strip():
|
|
||||||
text = block.text.strip()
|
|
||||||
style = (getattr(getattr(block, "style", None), "name", "") or "").lower()
|
|
||||||
if "role" in style and parts:
|
|
||||||
parts.append("")
|
|
||||||
parts.append(f"- {text}" if "bullet" in style else text)
|
|
||||||
return _check_extracted_size(_normalize_text("\n".join(parts)))
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_plain_text(data: bytes) -> str:
|
|
||||||
try:
|
try:
|
||||||
decoded = data.decode("utf-8-sig")
|
subprocess.run(
|
||||||
except UnicodeDecodeError as exc:
|
["taskkill", "/PID", str(process.pid), "/T", "/F"],
|
||||||
raise HTTPException(status_code=400, detail="The uploaded text file is not valid UTF-8.") from exc
|
stdin=subprocess.DEVNULL,
|
||||||
return _check_extracted_size(_normalize_text(decoded))
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
timeout=5,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
except (OSError, subprocess.SubprocessError):
|
||||||
|
process.kill()
|
||||||
|
|
||||||
|
|
||||||
|
def _raise_parser_error(code: str) -> None:
|
||||||
|
fallback = _PARSER_ERRORS["parser_failed"]
|
||||||
|
status_code, detail = _PARSER_ERRORS.get(code, fallback) if isinstance(code, str) else fallback
|
||||||
|
raise HTTPException(status_code=status_code, detail=detail)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_parser_child(extension: str, data: bytes) -> dict[str, object]:
|
||||||
|
if not _parser_capacity.acquire(blocking=False):
|
||||||
|
raise HTTPException(status_code=503, detail="Document extraction is busy. Try again shortly.")
|
||||||
|
try:
|
||||||
|
with tempfile.TemporaryDirectory(prefix="work-", dir=_prepare_parser_temp_root()) as temp_root:
|
||||||
|
input_path = Path(temp_root, "input.bin")
|
||||||
|
output_path = Path(temp_root, "result.json")
|
||||||
|
input_path.write_bytes(data)
|
||||||
|
command = [
|
||||||
|
sys.executable,
|
||||||
|
str(PARSER_CHILD_PATH),
|
||||||
|
"--input", str(input_path),
|
||||||
|
"--output", str(output_path),
|
||||||
|
"--extension", extension,
|
||||||
|
"--cpu-seconds", str(PARSER_CPU_SECONDS),
|
||||||
|
"--memory-bytes", str(PARSER_MEMORY_BYTES),
|
||||||
|
]
|
||||||
|
creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0
|
||||||
|
try:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
command,
|
||||||
|
cwd=temp_root,
|
||||||
|
env=_parser_environment(temp_root),
|
||||||
|
stdin=subprocess.DEVNULL,
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
start_new_session=os.name == "posix",
|
||||||
|
creationflags=creation_flags,
|
||||||
|
)
|
||||||
|
except OSError:
|
||||||
|
_raise_parser_error("parser_failed")
|
||||||
|
try:
|
||||||
|
process.wait(timeout=PARSER_TIMEOUT_SECONDS)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
_terminate_parser_process(process)
|
||||||
|
try:
|
||||||
|
process.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
process.kill()
|
||||||
|
raise HTTPException(status_code=504, detail="Document extraction timed out.") from exc
|
||||||
|
|
||||||
|
if not output_path.exists() or output_path.stat().st_size > PARSER_MAX_RESULT_BYTES:
|
||||||
|
_raise_parser_error("parser_failed")
|
||||||
|
try:
|
||||||
|
result = json.loads(output_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||||
|
_raise_parser_error("parser_failed")
|
||||||
|
if not isinstance(result, dict) or result.get("ok") is not True:
|
||||||
|
_raise_parser_error(result.get("code") if isinstance(result, dict) else "parser_failed")
|
||||||
|
text = result.get("text")
|
||||||
|
if not isinstance(text, str) or not text or len(text) > MAX_EXTRACT_CHARS:
|
||||||
|
_raise_parser_error("parser_failed")
|
||||||
|
return result
|
||||||
|
finally:
|
||||||
|
_parser_capacity.release()
|
||||||
|
|
||||||
|
|
||||||
@app.post("/extract-text")
|
@app.post("/extract-text")
|
||||||
@@ -1228,36 +1267,14 @@ async def extract_text(file: UploadFile = File(...)):
|
|||||||
raise HTTPException(status_code=400, detail="This file type is not supported for AI extraction.")
|
raise HTTPException(status_code=400, detail="This file type is not supported for AI extraction.")
|
||||||
_validate_file_signature(extension, data)
|
_validate_file_signature(extension, data)
|
||||||
|
|
||||||
try:
|
result = await asyncio.to_thread(_run_parser_child, extension, data)
|
||||||
if extension in {".txt", ".md"}:
|
text = result["text"]
|
||||||
text = _extract_plain_text(data)
|
|
||||||
ocr_used = False
|
|
||||||
page_count = None
|
|
||||||
elif extension == ".docx":
|
|
||||||
text = _extract_docx_text(data)
|
|
||||||
ocr_used = False
|
|
||||||
page_count = None
|
|
||||||
elif extension == ".pdf":
|
|
||||||
text, ocr_used, page_count = _extract_pdf_text(data)
|
|
||||||
elif extension in IMAGE_EXTENSIONS:
|
|
||||||
image = Image.open(io.BytesIO(data))
|
|
||||||
_check_image_size(image)
|
|
||||||
text = _ocr_image(image)
|
|
||||||
ocr_used = True
|
|
||||||
page_count = 1
|
|
||||||
except HTTPException:
|
|
||||||
raise
|
|
||||||
except Exception as exc:
|
|
||||||
raise HTTPException(status_code=422, detail="Document extraction failed.") from exc
|
|
||||||
|
|
||||||
if not text:
|
|
||||||
raise HTTPException(status_code=422, detail="AI extraction did not find readable text in the uploaded file.")
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"text": text,
|
"text": text,
|
||||||
"ocr_used": ocr_used,
|
"ocr_used": result.get("ocr_used", False),
|
||||||
"content_type": file.content_type,
|
"content_type": file.content_type,
|
||||||
"page_count": page_count,
|
"page_count": result.get("page_count"),
|
||||||
"characters": len(text),
|
"characters": len(text),
|
||||||
"file_name": filename,
|
"file_name": filename,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,253 @@
|
|||||||
|
"""Resource-bounded CV document decoder invoked only as a child process."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import zipfile
|
||||||
|
|
||||||
|
|
||||||
|
MAX_EXTRACT_CHARS = 200_000
|
||||||
|
MAX_PDF_PAGES = 40
|
||||||
|
MAX_DOCX_ENTRIES = 256
|
||||||
|
MAX_DOCX_UNCOMPRESSED_BYTES = 32 * 1024 * 1024
|
||||||
|
MAX_DOCX_ENTRY_BYTES = 8 * 1024 * 1024
|
||||||
|
MAX_DOCX_COMPRESSION_RATIO = 100
|
||||||
|
MAX_IMAGE_DIMENSION = 12_000
|
||||||
|
MAX_IMAGE_PIXELS = 40_000_000
|
||||||
|
MAX_PDF_PAGE_PIXELS = 12_000_000
|
||||||
|
MAX_PDF_OCR_PIXELS = 120_000_000
|
||||||
|
OCR_LANGUAGES = "eng"
|
||||||
|
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
|
||||||
|
|
||||||
|
|
||||||
|
class ParserError(Exception):
|
||||||
|
def __init__(self, code: str):
|
||||||
|
super().__init__(code)
|
||||||
|
self.code = code
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_resource_limits(cpu_seconds: int, memory_bytes: int) -> None:
|
||||||
|
if os.name != "posix":
|
||||||
|
return
|
||||||
|
import resource
|
||||||
|
|
||||||
|
resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds + 1))
|
||||||
|
resource.setrlimit(resource.RLIMIT_AS, (memory_bytes, memory_bytes))
|
||||||
|
resource.setrlimit(resource.RLIMIT_FSIZE, (1024 * 1024, 1024 * 1024))
|
||||||
|
resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64))
|
||||||
|
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_text(value: str) -> str:
|
||||||
|
value = value.replace("\x00", " ").replace("\r\n", "\n").replace("\r", "\n")
|
||||||
|
lines: list[str] = []
|
||||||
|
blank = False
|
||||||
|
for raw_line in value.split("\n"):
|
||||||
|
line = re.sub(r"[\t\f\v]+", " ", raw_line).strip()
|
||||||
|
if not line:
|
||||||
|
if lines and not blank:
|
||||||
|
lines.append("")
|
||||||
|
blank = True
|
||||||
|
continue
|
||||||
|
lines.append(line)
|
||||||
|
blank = False
|
||||||
|
return "\n".join(lines).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _check_extracted_size(text: str, limit: int = MAX_EXTRACT_CHARS) -> str:
|
||||||
|
if len(text) > limit:
|
||||||
|
raise ParserError("extracted_text_limit")
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def _check_image_size(image, *, pixel_limit: int = MAX_IMAGE_PIXELS) -> None:
|
||||||
|
width, height = image.size
|
||||||
|
if width <= 0 or height <= 0 or width > MAX_IMAGE_DIMENSION or height > MAX_IMAGE_DIMENSION:
|
||||||
|
raise ParserError("image_dimensions")
|
||||||
|
if width * height > pixel_limit:
|
||||||
|
raise ParserError("image_pixel_limit")
|
||||||
|
if getattr(image, "n_frames", 1) != 1:
|
||||||
|
raise ParserError("image_frames")
|
||||||
|
|
||||||
|
|
||||||
|
def validate_docx_container(
|
||||||
|
data: bytes,
|
||||||
|
*,
|
||||||
|
entry_limit: int = MAX_DOCX_ENTRIES,
|
||||||
|
total_limit: int = MAX_DOCX_UNCOMPRESSED_BYTES,
|
||||||
|
entry_size_limit: int = MAX_DOCX_ENTRY_BYTES,
|
||||||
|
ratio_limit: int = MAX_DOCX_COMPRESSION_RATIO,
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||||
|
entries = archive.infolist()
|
||||||
|
names = {entry.filename for entry in entries}
|
||||||
|
if "[Content_Types].xml" not in names or "word/document.xml" not in names:
|
||||||
|
raise ParserError("docx_signature")
|
||||||
|
if len(entries) > entry_limit:
|
||||||
|
raise ParserError("docx_entry_limit")
|
||||||
|
total_size = 0
|
||||||
|
for entry in entries:
|
||||||
|
total_size += entry.file_size
|
||||||
|
if entry.file_size > entry_size_limit:
|
||||||
|
raise ParserError("docx_entry_size")
|
||||||
|
if entry.file_size > 0 and entry.compress_size == 0:
|
||||||
|
raise ParserError("docx_ratio")
|
||||||
|
if entry.compress_size > 0 and entry.file_size / entry.compress_size > ratio_limit:
|
||||||
|
raise ParserError("docx_ratio")
|
||||||
|
if total_size > total_limit:
|
||||||
|
raise ParserError("docx_expanded_size")
|
||||||
|
except ParserError:
|
||||||
|
raise
|
||||||
|
except (zipfile.BadZipFile, OSError) as exc:
|
||||||
|
raise ParserError("docx_signature") from exc
|
||||||
|
|
||||||
|
|
||||||
|
def extract_plain_text(data: bytes) -> str:
|
||||||
|
try:
|
||||||
|
decoded = data.decode("utf-8-sig")
|
||||||
|
except UnicodeDecodeError as exc:
|
||||||
|
raise ParserError("text_encoding") from exc
|
||||||
|
return _check_extracted_size(normalize_text(decoded))
|
||||||
|
|
||||||
|
|
||||||
|
def extract_docx_text(data: bytes) -> str:
|
||||||
|
validate_docx_container(data)
|
||||||
|
from docx import Document
|
||||||
|
|
||||||
|
document = Document(io.BytesIO(data))
|
||||||
|
parts: list[str] = []
|
||||||
|
blocks = document.iter_inner_content() if hasattr(document, "iter_inner_content") else document.paragraphs
|
||||||
|
for block in blocks:
|
||||||
|
if hasattr(block, "rows"):
|
||||||
|
for row in block.rows:
|
||||||
|
cells = [cell.text.strip() for cell in row.cells if cell.text and cell.text.strip()]
|
||||||
|
if cells:
|
||||||
|
parts.append(" | ".join(cells))
|
||||||
|
elif getattr(block, "text", "").strip():
|
||||||
|
text = block.text.strip()
|
||||||
|
style = (getattr(getattr(block, "style", None), "name", "") or "").lower()
|
||||||
|
if "role" in style and parts:
|
||||||
|
parts.append("")
|
||||||
|
parts.append(f"- {text}" if "bullet" in style else text)
|
||||||
|
return _check_extracted_size(normalize_text("\n".join(parts)))
|
||||||
|
|
||||||
|
|
||||||
|
def _ocr_image(image) -> str:
|
||||||
|
import pytesseract
|
||||||
|
|
||||||
|
if image.mode not in ("RGB", "L"):
|
||||||
|
image = image.convert("RGB")
|
||||||
|
return normalize_text(pytesseract.image_to_string(image, lang=OCR_LANGUAGES))
|
||||||
|
|
||||||
|
|
||||||
|
def extract_image_text(data: bytes, *, pixel_limit: int = MAX_IMAGE_PIXELS) -> str:
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
with Image.open(io.BytesIO(data)) as image:
|
||||||
|
_check_image_size(image, pixel_limit=pixel_limit)
|
||||||
|
return _check_extracted_size(_ocr_image(image))
|
||||||
|
|
||||||
|
|
||||||
|
def extract_pdf_text(data: bytes, *, page_limit: int = MAX_PDF_PAGES) -> tuple[str, bool, int]:
|
||||||
|
from pypdf import PdfReader
|
||||||
|
|
||||||
|
page_count = 0
|
||||||
|
extracted_pages: list[str] = []
|
||||||
|
try:
|
||||||
|
reader = PdfReader(io.BytesIO(data))
|
||||||
|
page_count = len(reader.pages)
|
||||||
|
if page_count > page_limit:
|
||||||
|
raise ParserError("pdf_page_limit")
|
||||||
|
for page in reader.pages:
|
||||||
|
try:
|
||||||
|
extracted_pages.append(page.extract_text(extraction_mode="layout") or "")
|
||||||
|
except TypeError:
|
||||||
|
extracted_pages.append(page.extract_text() or "")
|
||||||
|
except ParserError:
|
||||||
|
raise
|
||||||
|
except Exception:
|
||||||
|
extracted_pages = []
|
||||||
|
|
||||||
|
text = _check_extracted_size(normalize_text("\n".join(extracted_pages)))
|
||||||
|
if len(text) >= 80:
|
||||||
|
return text, False, page_count
|
||||||
|
|
||||||
|
import fitz
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
document = fitz.open(stream=data, filetype="pdf")
|
||||||
|
try:
|
||||||
|
page_count = document.page_count
|
||||||
|
if page_count > page_limit:
|
||||||
|
raise ParserError("pdf_page_limit")
|
||||||
|
ocr_pages: list[str] = []
|
||||||
|
total_pixels = 0
|
||||||
|
for page in document:
|
||||||
|
pixmap = page.get_pixmap(matrix=fitz.Matrix(2, 2), alpha=False)
|
||||||
|
page_pixels = pixmap.width * pixmap.height
|
||||||
|
total_pixels += page_pixels
|
||||||
|
if page_pixels > MAX_PDF_PAGE_PIXELS or total_pixels > MAX_PDF_OCR_PIXELS:
|
||||||
|
raise ParserError("pdf_ocr_limit")
|
||||||
|
with Image.open(io.BytesIO(pixmap.tobytes("png"))) as image:
|
||||||
|
_check_image_size(image, pixel_limit=MAX_PDF_PAGE_PIXELS)
|
||||||
|
ocr_pages.append(_ocr_image(image))
|
||||||
|
_check_extracted_size("\n".join(ocr_pages))
|
||||||
|
return _check_extracted_size(normalize_text("\n".join(ocr_pages))), True, page_count
|
||||||
|
finally:
|
||||||
|
document.close()
|
||||||
|
|
||||||
|
|
||||||
|
def extract(extension: str, data: bytes) -> dict[str, object]:
|
||||||
|
if extension in {".txt", ".md"}:
|
||||||
|
text = extract_plain_text(data)
|
||||||
|
return {"text": text, "ocr_used": False, "page_count": None}
|
||||||
|
if extension == ".docx":
|
||||||
|
text = extract_docx_text(data)
|
||||||
|
return {"text": text, "ocr_used": False, "page_count": None}
|
||||||
|
if extension == ".pdf":
|
||||||
|
text, ocr_used, page_count = extract_pdf_text(data)
|
||||||
|
return {"text": text, "ocr_used": ocr_used, "page_count": page_count}
|
||||||
|
if extension in IMAGE_EXTENSIONS:
|
||||||
|
text = extract_image_text(data)
|
||||||
|
return {"text": text, "ocr_used": True, "page_count": 1}
|
||||||
|
raise ParserError("unsupported_type")
|
||||||
|
|
||||||
|
|
||||||
|
def _write_result(path: Path, result: dict[str, object]) -> None:
|
||||||
|
path.write_text(json.dumps(result, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser(add_help=False)
|
||||||
|
parser.add_argument("--input", required=True)
|
||||||
|
parser.add_argument("--output", required=True)
|
||||||
|
parser.add_argument("--extension", required=True)
|
||||||
|
parser.add_argument("--cpu-seconds", type=int, default=20)
|
||||||
|
parser.add_argument("--memory-bytes", type=int, default=512 * 1024 * 1024)
|
||||||
|
args = parser.parse_args()
|
||||||
|
output_path = Path(args.output)
|
||||||
|
try:
|
||||||
|
_apply_resource_limits(args.cpu_seconds, args.memory_bytes)
|
||||||
|
data = Path(args.input).read_bytes()
|
||||||
|
result = extract(args.extension.lower(), data)
|
||||||
|
if not result.get("text"):
|
||||||
|
raise ParserError("no_text")
|
||||||
|
_write_result(output_path, {"ok": True, **result})
|
||||||
|
return 0
|
||||||
|
except ParserError as exc:
|
||||||
|
_write_result(output_path, {"ok": False, "code": exc.code})
|
||||||
|
return 2
|
||||||
|
except BaseException:
|
||||||
|
_write_result(output_path, {"ok": False, "code": "parser_failed"})
|
||||||
|
return 3
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -1,3 +1,3 @@
|
|||||||
-r requirements.txt
|
-r requirements.txt
|
||||||
pytest==8.3.5
|
pytest==8.3.5
|
||||||
httpx==0.28.1
|
httpx2==2.12.0
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
import importlib
|
import importlib
|
||||||
import io
|
import io
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
import textwrap
|
||||||
import zipfile
|
import zipfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from docx import Document
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from pypdf import PdfWriter
|
from pypdf import PdfWriter
|
||||||
@@ -67,6 +71,10 @@ def test_health_reports_runtime_without_ollama_and_without_forcing_model_load(mo
|
|||||||
assert payload["model_loaded"] is False
|
assert payload["model_loaded"] is False
|
||||||
assert payload["model_disabled"] is True
|
assert payload["model_disabled"] is True
|
||||||
assert payload["summarize_available"] is False
|
assert payload["summarize_available"] is False
|
||||||
|
assert payload["parser_isolated"] is True
|
||||||
|
assert payload["parser_concurrency"] == 1
|
||||||
|
assert payload["parser_timeout_seconds"] == 25
|
||||||
|
assert payload["parser_memory_mib"] == 512
|
||||||
assert "disabled" in payload["model_load_error"].lower()
|
assert "disabled" in payload["model_load_error"].lower()
|
||||||
assert payload["ollama_configured"] is False
|
assert payload["ollama_configured"] is False
|
||||||
assert payload["ollama_model"] is None
|
assert payload["ollama_model"] is None
|
||||||
@@ -517,7 +525,7 @@ def test_health_stays_open_so_probes_and_healthchecks_work(monkeypatch):
|
|||||||
def test_extract_text_rejects_oversized_upload_before_parsing(monkeypatch):
|
def test_extract_text_rejects_oversized_upload_before_parsing(monkeypatch):
|
||||||
module = load_app_module(monkeypatch)
|
module = load_app_module(monkeypatch)
|
||||||
monkeypatch.setattr(module, "MAX_EXTRACT_FILE_BYTES", 32)
|
monkeypatch.setattr(module, "MAX_EXTRACT_FILE_BYTES", 32)
|
||||||
monkeypatch.setattr(module, "_extract_plain_text", lambda data: (_ for _ in ()).throw(AssertionError("parser must not run")))
|
monkeypatch.setattr(module, "_run_parser_child", lambda extension, data: (_ for _ in ()).throw(AssertionError("parser must not run")))
|
||||||
client = TestClient(module.app)
|
client = TestClient(module.app)
|
||||||
|
|
||||||
response = client.post("/extract-text", files={"file": ("large.txt", b"x" * 64, "text/plain")})
|
response = client.post("/extract-text", files={"file": ("large.txt", b"x" * 64, "text/plain")})
|
||||||
@@ -547,33 +555,36 @@ def test_extract_text_rejects_binary_plain_text(monkeypatch):
|
|||||||
|
|
||||||
|
|
||||||
def test_extract_text_rejects_pdf_over_page_limit(monkeypatch):
|
def test_extract_text_rejects_pdf_over_page_limit(monkeypatch):
|
||||||
module = load_app_module(monkeypatch)
|
load_app_module(monkeypatch)
|
||||||
monkeypatch.setattr(module, "MAX_PDF_PAGES", 1)
|
import parser_child
|
||||||
|
|
||||||
writer = PdfWriter()
|
writer = PdfWriter()
|
||||||
writer.add_blank_page(width=100, height=100)
|
writer.add_blank_page(width=100, height=100)
|
||||||
writer.add_blank_page(width=100, height=100)
|
writer.add_blank_page(width=100, height=100)
|
||||||
payload = io.BytesIO()
|
payload = io.BytesIO()
|
||||||
writer.write(payload)
|
writer.write(payload)
|
||||||
client = TestClient(module.app)
|
try:
|
||||||
|
parser_child.extract_pdf_text(payload.getvalue(), page_limit=1)
|
||||||
response = client.post("/extract-text", files={"file": ("resume.pdf", payload.getvalue(), "application/pdf")})
|
except parser_child.ParserError as exc:
|
||||||
|
assert exc.code == "pdf_page_limit"
|
||||||
assert response.status_code == 422
|
else:
|
||||||
assert response.json()["detail"] == "The PDF contains too many pages."
|
raise AssertionError("Expected the PDF page limit to reject the document")
|
||||||
|
|
||||||
|
|
||||||
def test_extract_text_rejects_image_pixel_limit_before_ocr(monkeypatch):
|
def test_extract_text_rejects_image_pixel_limit_before_ocr(monkeypatch):
|
||||||
module = load_app_module(monkeypatch)
|
load_app_module(monkeypatch)
|
||||||
monkeypatch.setattr(module, "MAX_IMAGE_PIXELS", 50)
|
import parser_child
|
||||||
monkeypatch.setattr(module, "_ocr_image", lambda image: (_ for _ in ()).throw(AssertionError("OCR must not run")))
|
|
||||||
|
monkeypatch.setattr(parser_child, "_ocr_image", lambda image: (_ for _ in ()).throw(AssertionError("OCR must not run")))
|
||||||
payload = io.BytesIO()
|
payload = io.BytesIO()
|
||||||
Image.new("RGB", (10, 10), "white").save(payload, format="PNG")
|
Image.new("RGB", (10, 10), "white").save(payload, format="PNG")
|
||||||
client = TestClient(module.app)
|
|
||||||
|
|
||||||
response = client.post("/extract-text", files={"file": ("resume.png", payload.getvalue(), "image/png")})
|
try:
|
||||||
|
parser_child.extract_image_text(payload.getvalue(), pixel_limit=50)
|
||||||
assert response.status_code == 422
|
except parser_child.ParserError as exc:
|
||||||
assert response.json()["detail"] == "The document image contains too many pixels."
|
assert exc.code == "image_pixel_limit"
|
||||||
|
else:
|
||||||
|
raise AssertionError("Expected the image pixel limit to reject the document")
|
||||||
|
|
||||||
|
|
||||||
def test_docx_container_rejects_excessive_entries(monkeypatch):
|
def test_docx_container_rejects_excessive_entries(monkeypatch):
|
||||||
@@ -596,7 +607,7 @@ def test_docx_container_rejects_excessive_entries(monkeypatch):
|
|||||||
|
|
||||||
def test_extract_text_does_not_leak_parser_exception(monkeypatch):
|
def test_extract_text_does_not_leak_parser_exception(monkeypatch):
|
||||||
module = load_app_module(monkeypatch)
|
module = load_app_module(monkeypatch)
|
||||||
monkeypatch.setattr(module, "_extract_pdf_text", lambda data: (_ for _ in ()).throw(RuntimeError("private path C:/secret")))
|
monkeypatch.setattr(module, "_run_parser_child", lambda extension, data: module._raise_parser_error("parser_failed"))
|
||||||
client = TestClient(module.app)
|
client = TestClient(module.app)
|
||||||
|
|
||||||
response = client.post("/extract-text", files={"file": ("resume.pdf", b"%PDF-invalid", "application/pdf")})
|
response = client.post("/extract-text", files={"file": ("resume.pdf", b"%PDF-invalid", "application/pdf")})
|
||||||
@@ -607,16 +618,19 @@ def test_extract_text_does_not_leak_parser_exception(monkeypatch):
|
|||||||
|
|
||||||
|
|
||||||
def test_extraction_normalization_preserves_sections_and_bullets(monkeypatch):
|
def test_extraction_normalization_preserves_sections_and_bullets(monkeypatch):
|
||||||
module = load_app_module(monkeypatch)
|
load_app_module(monkeypatch)
|
||||||
|
import parser_child
|
||||||
|
|
||||||
normalized = module._normalize_text("PROFILE\n\nEngineer\n• Built APIs\n• Shipped services")
|
normalized = parser_child.normalize_text("PROFILE\n\nEngineer\n• Built APIs\n• Shipped services")
|
||||||
|
|
||||||
assert normalized.splitlines() == ["PROFILE", "", "Engineer", "• Built APIs", "• Shipped services"]
|
assert normalized.splitlines() == ["PROFILE", "", "Engineer", "• Built APIs", "• Shipped services"]
|
||||||
|
|
||||||
|
|
||||||
def test_docx_extraction_preserves_paragraph_and_table_boundaries(monkeypatch):
|
def test_docx_extraction_preserves_paragraph_and_table_boundaries(monkeypatch):
|
||||||
module = load_app_module(monkeypatch)
|
load_app_module(monkeypatch)
|
||||||
document = module.Document()
|
import parser_child
|
||||||
|
|
||||||
|
document = Document()
|
||||||
document.add_heading("Technical Skills", level=1)
|
document.add_heading("Technical Skills", level=1)
|
||||||
table = document.add_table(rows=2, cols=2)
|
table = document.add_table(rows=2, cols=2)
|
||||||
table.cell(0, 0).text = "Backend"
|
table.cell(0, 0).text = "Backend"
|
||||||
@@ -626,13 +640,129 @@ def test_docx_extraction_preserves_paragraph_and_table_boundaries(monkeypatch):
|
|||||||
payload = io.BytesIO()
|
payload = io.BytesIO()
|
||||||
document.save(payload)
|
document.save(payload)
|
||||||
|
|
||||||
extracted = module._extract_docx_text(payload.getvalue())
|
extracted = parser_child.extract_docx_text(payload.getvalue())
|
||||||
|
|
||||||
assert "Technical Skills" in extracted
|
assert "Technical Skills" in extracted
|
||||||
assert "Backend | C#, .NET" in extracted
|
assert "Backend | C#, .NET" in extracted
|
||||||
assert "DevOps | Docker, Linux" in extracted
|
assert "DevOps | Docker, Linux" in extracted
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_text_runs_in_isolated_child(monkeypatch):
|
||||||
|
module = load_app_module(monkeypatch)
|
||||||
|
client = TestClient(module.app)
|
||||||
|
|
||||||
|
response = client.post(
|
||||||
|
"/extract-text",
|
||||||
|
files={"file": ("resume.md", b"# Ada Lovelace\n\n## Skills\nC#", "text/markdown")},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert response.json()["text"] == "# Ada Lovelace\n\n## Skills\nC#"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_child_receives_no_provider_secrets(monkeypatch, tmp_path):
|
||||||
|
module = load_app_module(monkeypatch)
|
||||||
|
monkeypatch.setenv("GEMINI_API_KEY", "must-not-leak")
|
||||||
|
monkeypatch.setenv("GROQ_API_KEY", "must-not-leak")
|
||||||
|
child = tmp_path / "environment_probe.py"
|
||||||
|
child.write_text(textwrap.dedent("""
|
||||||
|
import argparse, json, os
|
||||||
|
from pathlib import Path
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('--input')
|
||||||
|
parser.add_argument('--output')
|
||||||
|
parser.add_argument('--extension')
|
||||||
|
parser.add_argument('--cpu-seconds')
|
||||||
|
parser.add_argument('--memory-bytes')
|
||||||
|
args = parser.parse_args()
|
||||||
|
leaked = any(os.environ.get(key) for key in ('GEMINI_API_KEY', 'GROQ_API_KEY', 'AI_SERVICE_TOKEN'))
|
||||||
|
Path(args.output).write_text(json.dumps({'ok': not leaked, 'text': 'safe', 'ocr_used': False, 'page_count': None} if not leaked else {'ok': False, 'code': 'parser_failed'}), encoding='utf-8')
|
||||||
|
"""), encoding="utf-8")
|
||||||
|
monkeypatch.setattr(module, "PARSER_CHILD_PATH", child)
|
||||||
|
|
||||||
|
assert module._run_parser_child(".txt", b"safe")["text"] == "safe"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_child_timeout_is_stable_and_releases_capacity(monkeypatch, tmp_path):
|
||||||
|
module = load_app_module(monkeypatch)
|
||||||
|
child = tmp_path / "sleeping_parser.py"
|
||||||
|
child.write_text("import time\ntime.sleep(30)\n", encoding="utf-8")
|
||||||
|
monkeypatch.setattr(module, "PARSER_CHILD_PATH", child)
|
||||||
|
monkeypatch.setattr(module, "PARSER_TIMEOUT_SECONDS", 0.1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
module._run_parser_child(".txt", b"safe")
|
||||||
|
except module.HTTPException as exc:
|
||||||
|
assert exc.status_code == 504
|
||||||
|
assert exc.detail == "Document extraction timed out."
|
||||||
|
else:
|
||||||
|
raise AssertionError("Expected the parser deadline to terminate the child")
|
||||||
|
|
||||||
|
assert module._parser_capacity.acquire(blocking=False)
|
||||||
|
module._parser_capacity.release()
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_timeout_terminates_descendants(monkeypatch, tmp_path):
|
||||||
|
module = load_app_module(monkeypatch)
|
||||||
|
pid_path = tmp_path / "descendant.pid"
|
||||||
|
child = tmp_path / "parser_with_descendant.py"
|
||||||
|
child.write_text(textwrap.dedent(f"""
|
||||||
|
import subprocess, sys, time
|
||||||
|
from pathlib import Path
|
||||||
|
descendant = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])
|
||||||
|
Path({str(pid_path)!r}).write_text(str(descendant.pid), encoding='ascii')
|
||||||
|
time.sleep(30)
|
||||||
|
"""), encoding="utf-8")
|
||||||
|
monkeypatch.setattr(module, "PARSER_CHILD_PATH", child)
|
||||||
|
monkeypatch.setattr(module, "PARSER_TIMEOUT_SECONDS", 0.2)
|
||||||
|
|
||||||
|
try:
|
||||||
|
module._run_parser_child(".txt", b"safe")
|
||||||
|
except module.HTTPException as exc:
|
||||||
|
assert exc.status_code == 504
|
||||||
|
else:
|
||||||
|
raise AssertionError("Expected the parser deadline to terminate the process tree")
|
||||||
|
|
||||||
|
descendant_pid = int(pid_path.read_text(encoding="ascii"))
|
||||||
|
if os.name == "nt":
|
||||||
|
listing = subprocess.run(
|
||||||
|
["tasklist", "/FI", f"PID eq {descendant_pid}", "/FO", "CSV", "/NH"],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
).stdout
|
||||||
|
assert f'"{descendant_pid}"' not in listing
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
os.kill(descendant_pid, 0)
|
||||||
|
except ProcessLookupError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
os.kill(descendant_pid, 9)
|
||||||
|
raise AssertionError("Parser descendant survived the parent deadline")
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_temp_cleanup_removes_only_stale_work_directories(monkeypatch, tmp_path):
|
||||||
|
module = load_app_module(monkeypatch)
|
||||||
|
parser_root = tmp_path / "parser-root"
|
||||||
|
stale = parser_root / "work-stale"
|
||||||
|
recent = parser_root / "work-recent"
|
||||||
|
unrelated = parser_root / "keep-me"
|
||||||
|
stale.mkdir(parents=True)
|
||||||
|
recent.mkdir()
|
||||||
|
unrelated.mkdir()
|
||||||
|
old = 1_000_000
|
||||||
|
os.utime(stale, (old, old))
|
||||||
|
monkeypatch.setattr(module, "PARSER_TEMP_ROOT", parser_root.resolve())
|
||||||
|
monkeypatch.setattr(module, "PARSER_STALE_WORK_SECONDS", 300)
|
||||||
|
monkeypatch.setattr(module, "_parser_temp_ready", False)
|
||||||
|
|
||||||
|
assert module._prepare_parser_temp_root() == parser_root.resolve()
|
||||||
|
assert not stale.exists()
|
||||||
|
assert recent.exists()
|
||||||
|
assert unrelated.exists()
|
||||||
|
|
||||||
|
|
||||||
def test_cache_purge_requires_service_token_and_clears_content(monkeypatch):
|
def test_cache_purge_requires_service_token_and_clears_content(monkeypatch):
|
||||||
module = load_app_module(monkeypatch, service_token="s3cret")
|
module = load_app_module(monkeypatch, service_token="s3cret")
|
||||||
module.cache["synthetic-key"] = "synthetic-summary"
|
module.cache["synthetic-key"] = "synthetic-summary"
|
||||||
|
|||||||
Reference in New Issue
Block a user