cesnimda/jobtrackingapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(cv): isolate document parsing

Browse Source 
Run untrusted document decoders in a secret-free, resource-bounded child process and terminate its process tree on deadline. Harden the production container and enforce parser and lint gates in CI.
This commit is contained in:
cesnimda
2026-08-30 11:12:25 +02:00
parent a8bf505ce5
commit 19c5251612
13 changed files with 677 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/summarizer/tests/test_app.py
+153 -23
View File
@@ -1,10 +1,14 @@
import importlib
import io
import json
import os
import subprocess
import sys
import textwrap
import zipfile
from pathlib import Path
from docx import Document
from fastapi.testclient import TestClient
from PIL import Image
from pypdf import PdfWriter
@@ -67,6 +71,10 @@ def test_health_reports_runtime_without_ollama_and_without_forcing_model_load(mo
assert payload["model_loaded"] is False
assert payload["model_disabled"] is True
assert payload["summarize_available"] is False
assert payload["parser_isolated"] is True
assert payload["parser_concurrency"] == 1
assert payload["parser_timeout_seconds"] == 25
assert payload["parser_memory_mib"] == 512
assert "disabled" in payload["model_load_error"].lower()
assert payload["ollama_configured"] is False
assert payload["ollama_model"] is None
@@ -517,7 +525,7 @@ def test_health_stays_open_so_probes_and_healthchecks_work(monkeypatch):
def test_extract_text_rejects_oversized_upload_before_parsing(monkeypatch):
module = load_app_module(monkeypatch)
monkeypatch.setattr(module, "MAX_EXTRACT_FILE_BYTES", 32)
monkeypatch.setattr(module, "_extract_plain_text", lambda data: (_ for _ in ()).throw(AssertionError("parser must not run")))
monkeypatch.setattr(module, "_run_parser_child", lambda extension, data: (_ for _ in ()).throw(AssertionError("parser must not run")))
client = TestClient(module.app)
response = client.post("/extract-text", files={"file": ("large.txt", b"x" * 64, "text/plain")})
@@ -547,33 +555,36 @@ def test_extract_text_rejects_binary_plain_text(monkeypatch):
def test_extract_text_rejects_pdf_over_page_limit(monkeypatch):
module = load_app_module(monkeypatch)
monkeypatch.setattr(module, "MAX_PDF_PAGES", 1)
load_app_module(monkeypatch)
import parser_child
writer = PdfWriter()
writer.add_blank_page(width=100, height=100)
writer.add_blank_page(width=100, height=100)
payload = io.BytesIO()
writer.write(payload)
client = TestClient(module.app)
response = client.post("/extract-text", files={"file": ("resume.pdf", payload.getvalue(), "application/pdf")})
assert response.status_code == 422
assert response.json()["detail"] == "The PDF contains too many pages."
try:
parser_child.extract_pdf_text(payload.getvalue(), page_limit=1)
except parser_child.ParserError as exc:
assert exc.code == "pdf_page_limit"
else:
raise AssertionError("Expected the PDF page limit to reject the document")
def test_extract_text_rejects_image_pixel_limit_before_ocr(monkeypatch):
module = load_app_module(monkeypatch)
monkeypatch.setattr(module, "MAX_IMAGE_PIXELS", 50)
monkeypatch.setattr(module, "_ocr_image", lambda image: (_ for _ in ()).throw(AssertionError("OCR must not run")))
load_app_module(monkeypatch)
import parser_child
monkeypatch.setattr(parser_child, "_ocr_image", lambda image: (_ for _ in ()).throw(AssertionError("OCR must not run")))
payload = io.BytesIO()
Image.new("RGB", (10, 10), "white").save(payload, format="PNG")
client = TestClient(module.app)
response = client.post("/extract-text", files={"file": ("resume.png", payload.getvalue(), "image/png")})
assert response.status_code == 422
assert response.json()["detail"] == "The document image contains too many pixels."
try:
parser_child.extract_image_text(payload.getvalue(), pixel_limit=50)
except parser_child.ParserError as exc:
assert exc.code == "image_pixel_limit"
else:
raise AssertionError("Expected the image pixel limit to reject the document")
def test_docx_container_rejects_excessive_entries(monkeypatch):
@@ -596,7 +607,7 @@ def test_docx_container_rejects_excessive_entries(monkeypatch):
def test_extract_text_does_not_leak_parser_exception(monkeypatch):
module = load_app_module(monkeypatch)
monkeypatch.setattr(module, "_extract_pdf_text", lambda data: (_ for _ in ()).throw(RuntimeError("private path C:/secret")))
monkeypatch.setattr(module, "_run_parser_child", lambda extension, data: module._raise_parser_error("parser_failed"))
client = TestClient(module.app)
response = client.post("/extract-text", files={"file": ("resume.pdf", b"%PDF-invalid", "application/pdf")})
@@ -607,16 +618,19 @@ def test_extract_text_does_not_leak_parser_exception(monkeypatch):
def test_extraction_normalization_preserves_sections_and_bullets(monkeypatch):
module = load_app_module(monkeypatch)
load_app_module(monkeypatch)
import parser_child
normalized = module._normalize_text("PROFILE\n\nEngineer\n• Built APIs\n• Shipped services")
normalized = parser_child.normalize_text("PROFILE\n\nEngineer\n• Built APIs\n• Shipped services")
assert normalized.splitlines() == ["PROFILE", "", "Engineer", "• Built APIs", "• Shipped services"]
def test_docx_extraction_preserves_paragraph_and_table_boundaries(monkeypatch):
module = load_app_module(monkeypatch)
document = module.Document()
load_app_module(monkeypatch)
import parser_child
document = Document()
document.add_heading("Technical Skills", level=1)
table = document.add_table(rows=2, cols=2)
table.cell(0, 0).text = "Backend"
@@ -626,13 +640,129 @@ def test_docx_extraction_preserves_paragraph_and_table_boundaries(monkeypatch):
payload = io.BytesIO()
document.save(payload)
extracted = module._extract_docx_text(payload.getvalue())
extracted = parser_child.extract_docx_text(payload.getvalue())
assert "Technical Skills" in extracted
assert "Backend | C#, .NET" in extracted
assert "DevOps | Docker, Linux" in extracted
def test_extract_text_runs_in_isolated_child(monkeypatch):
module = load_app_module(monkeypatch)
client = TestClient(module.app)
response = client.post(
"/extract-text",
files={"file": ("resume.md", b"# Ada Lovelace\n\n## Skills\nC#", "text/markdown")},
)
assert response.status_code == 200
assert response.json()["text"] == "# Ada Lovelace\n\n## Skills\nC#"
def test_parser_child_receives_no_provider_secrets(monkeypatch, tmp_path):
module = load_app_module(monkeypatch)
monkeypatch.setenv("GEMINI_API_KEY", "must-not-leak")
monkeypatch.setenv("GROQ_API_KEY", "must-not-leak")
child = tmp_path / "environment_probe.py"
child.write_text(textwrap.dedent("""
import argparse, json, os
from pathlib import Path
parser = argparse.ArgumentParser()
parser.add_argument('--input')
parser.add_argument('--output')
parser.add_argument('--extension')
parser.add_argument('--cpu-seconds')
parser.add_argument('--memory-bytes')
args = parser.parse_args()
leaked = any(os.environ.get(key) for key in ('GEMINI_API_KEY', 'GROQ_API_KEY', 'AI_SERVICE_TOKEN'))
Path(args.output).write_text(json.dumps({'ok': not leaked, 'text': 'safe', 'ocr_used': False, 'page_count': None} if not leaked else {'ok': False, 'code': 'parser_failed'}), encoding='utf-8')
"""), encoding="utf-8")
monkeypatch.setattr(module, "PARSER_CHILD_PATH", child)
assert module._run_parser_child(".txt", b"safe")["text"] == "safe"
def test_parser_child_timeout_is_stable_and_releases_capacity(monkeypatch, tmp_path):
module = load_app_module(monkeypatch)
child = tmp_path / "sleeping_parser.py"
child.write_text("import time\ntime.sleep(30)\n", encoding="utf-8")
monkeypatch.setattr(module, "PARSER_CHILD_PATH", child)
monkeypatch.setattr(module, "PARSER_TIMEOUT_SECONDS", 0.1)
try:
module._run_parser_child(".txt", b"safe")
except module.HTTPException as exc:
assert exc.status_code == 504
assert exc.detail == "Document extraction timed out."
else:
raise AssertionError("Expected the parser deadline to terminate the child")
assert module._parser_capacity.acquire(blocking=False)
module._parser_capacity.release()
def test_parser_timeout_terminates_descendants(monkeypatch, tmp_path):
module = load_app_module(monkeypatch)
pid_path = tmp_path / "descendant.pid"
child = tmp_path / "parser_with_descendant.py"
child.write_text(textwrap.dedent(f"""
import subprocess, sys, time
from pathlib import Path
descendant = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)'])
Path({str(pid_path)!r}).write_text(str(descendant.pid), encoding='ascii')
time.sleep(30)
"""), encoding="utf-8")
monkeypatch.setattr(module, "PARSER_CHILD_PATH", child)
monkeypatch.setattr(module, "PARSER_TIMEOUT_SECONDS", 0.2)
try:
module._run_parser_child(".txt", b"safe")
except module.HTTPException as exc:
assert exc.status_code == 504
else:
raise AssertionError("Expected the parser deadline to terminate the process tree")
descendant_pid = int(pid_path.read_text(encoding="ascii"))
if os.name == "nt":
listing = subprocess.run(
["tasklist", "/FI", f"PID eq {descendant_pid}", "/FO", "CSV", "/NH"],
capture_output=True,
text=True,
check=False,
).stdout
assert f'"{descendant_pid}"' not in listing
else:
try:
os.kill(descendant_pid, 0)
except ProcessLookupError:
pass
else:
os.kill(descendant_pid, 9)
raise AssertionError("Parser descendant survived the parent deadline")
def test_parser_temp_cleanup_removes_only_stale_work_directories(monkeypatch, tmp_path):
module = load_app_module(monkeypatch)
parser_root = tmp_path / "parser-root"
stale = parser_root / "work-stale"
recent = parser_root / "work-recent"
unrelated = parser_root / "keep-me"
stale.mkdir(parents=True)
recent.mkdir()
unrelated.mkdir()
old = 1_000_000
os.utime(stale, (old, old))
monkeypatch.setattr(module, "PARSER_TEMP_ROOT", parser_root.resolve())
monkeypatch.setattr(module, "PARSER_STALE_WORK_SECONDS", 300)
monkeypatch.setattr(module, "_parser_temp_ready", False)
assert module._prepare_parser_temp_root() == parser_root.resolve()
assert not stale.exists()
assert recent.exists()
assert unrelated.exists()
def test_cache_purge_requires_service_token_and_clears_content(monkeypatch):
module = load_app_module(monkeypatch, service_token="s3cret")
module.cache["synthetic-key"] = "synthetic-summary"