cesnimda/jobtrackingapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cv-import): add bounded model-free Docling extraction

Browse Source
This commit is contained in:
cesnimda
2026-10-26 14:17:43 +01:00
parent 731536db26
commit 3ea8ff9d62
7 changed files with 1593 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/summarizer/app.py
+65 -10
View File
@@ -110,6 +110,10 @@ PARSER_CHILD_PATH = Path(__file__).with_name("parser_child.py")
PARSER_TIMEOUT_SECONDS = max(5, min(int(os.getenv("PARSER_TIMEOUT_SECONDS", "25")), 120))
PARSER_CPU_SECONDS = max(2, min(int(os.getenv("PARSER_CPU_SECONDS", "20")), PARSER_TIMEOUT_SECONDS))
PARSER_MEMORY_BYTES = max(256, min(int(os.getenv("PARSER_MEMORY_MIB", "512")), 2048)) * 1024 * 1024
CV_IMPORT_PYTHON = os.getenv("CV_IMPORT_PYTHON", sys.executable)
CV_IMPORT_TIMEOUT_SECONDS = max(5, min(int(os.getenv("CV_IMPORT_TIMEOUT_SECONDS", "40")), 120))
CV_IMPORT_CPU_SECONDS = max(2, min(int(os.getenv("CV_IMPORT_CPU_SECONDS", "30")), CV_IMPORT_TIMEOUT_SECONDS))
CV_IMPORT_MEMORY_BYTES = max(512, min(int(os.getenv("CV_IMPORT_MEMORY_MIB", "1024")), 2048)) * 1024 * 1024
PARSER_MAX_RESULT_BYTES = 1024 * 1024
_parser_capacity = threading.BoundedSemaphore(value=1)
PARSER_TEMP_ROOT = Path(os.getenv("PARSER_TEMP_ROOT", tempfile.gettempdir()), "jobtracker-cv-parser").resolve()
@@ -1184,6 +1188,15 @@ _PARSER_ERRORS = {
"docx_entry_size": (422, "The DOCX contains an oversized file."),
"docx_ratio": (422, "The DOCX compression ratio is not supported."),
"docx_expanded_size": (422, "The DOCX expands beyond the supported size."),
"docx_unsafe_archive": (422, "The DOCX archive structure is not supported."),
"pdf_signature": (400, "The uploaded file does not match its PDF extension."),
"pdf_invalid": (422, "The PDF is malformed or unsupported."),
"pdf_encrypted": (422, "Encrypted PDFs are not supported."),
"pdf_object_limit": (422, "The PDF structure exceeds the supported limits."),
"external_content": (422, "Active or external document content is not supported."),
"block_limit": (422, "The document contains too many text blocks."),
"file_size_limit": (400, "The uploaded file size is not supported."),
"docling_unavailable": (503, "Document conversion runtime is unavailable."),
"pdf_page_limit": (422, "The PDF contains too many pages."),
"pdf_ocr_limit": (422, "The PDF requires too much OCR processing."),
"image_dimensions": (422, "The document image dimensions are not supported."),
@@ -1203,6 +1216,9 @@ def _parser_environment(temp_root: str) -> dict[str, str]:
"PYTHONDONTWRITEBYTECODE": "1",
"OMP_NUM_THREADS": "1",
"OPENBLAS_NUM_THREADS": "1",
"HF_HUB_OFFLINE": "1",
"TRANSFORMERS_OFFLINE": "1",
"HF_HUB_DISABLE_TELEMETRY": "1",
"TMPDIR": temp_root,
"TEMP": temp_root,
"TMP": temp_root,
@@ -1259,7 +1275,7 @@ def _raise_parser_error(code: str) -> None:
raise HTTPException(status_code=status_code, detail=detail)
def _run_parser_child(extension: str, data: bytes) -> dict[str, object]:
def _run_parser_child(extension: str, data: bytes, *, document: bool = False) -> dict[str, object]:
if not _parser_capacity.acquire(blocking=False):
raise HTTPException(status_code=503, detail="Document extraction is busy. Try again shortly.")
try:
@@ -1268,14 +1284,16 @@ def _run_parser_child(extension: str, data: bytes) -> dict[str, object]:
output_path = Path(temp_root, "result.json")
input_path.write_bytes(data)
command = [
sys.executable,
CV_IMPORT_PYTHON if document else sys.executable,
str(PARSER_CHILD_PATH),
"--input", str(input_path),
"--output", str(output_path),
"--extension", extension,
"--cpu-seconds", str(PARSER_CPU_SECONDS),
"--memory-bytes", str(PARSER_MEMORY_BYTES),
"--cpu-seconds", str(CV_IMPORT_CPU_SECONDS if document else PARSER_CPU_SECONDS),
"--memory-bytes", str(CV_IMPORT_MEMORY_BYTES if document else PARSER_MEMORY_BYTES),
]
if document:
command.extend(["--mode", "document"])
creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0
try:
process = subprocess.Popen(
@@ -1291,7 +1309,7 @@ def _run_parser_child(extension: str, data: bytes) -> dict[str, object]:
except OSError:
_raise_parser_error("parser_failed")
try:
process.wait(timeout=PARSER_TIMEOUT_SECONDS)
process.wait(timeout=CV_IMPORT_TIMEOUT_SECONDS if document else PARSER_TIMEOUT_SECONDS)
except subprocess.TimeoutExpired as exc:
_terminate_parser_process(process)
try:
@@ -1311,13 +1329,35 @@ def _run_parser_child(extension: str, data: bytes) -> dict[str, object]:
text = result.get("text")
if not isinstance(text, str) or not text or len(text) > MAX_EXTRACT_CHARS:
_raise_parser_error("parser_failed")
if document:
_validate_document_result(result)
return result
finally:
_parser_capacity.release()
@app.post("/extract-text")
async def extract_text(file: UploadFile = File(...)):
def _validate_document_result(result: dict[str, object]) -> None:
blocks = result.get("blocks")
warnings = result.get("warnings")
page_count = result.get("page_count")
if (result.get("method") not in {"docling-docx-v1", "docling-native-pdf-v1"}
or result.get("ocr_used") is not False
or not isinstance(blocks, list) or not 1 <= len(blocks) <= 5000
or not isinstance(warnings, list) or len(warnings) > 16
or any(not isinstance(warning, str) or not re.fullmatch(r"[a-z0-9_]{1,100}", warning) for warning in warnings)
or (page_count is not None and (type(page_count) is not int or not 1 <= page_count <= 40))):
_raise_parser_error("parser_failed")
for index, block in enumerate(blocks):
if (not isinstance(block, dict) or block.get("id") != f"b{index + 1:04d}"
or not isinstance(block.get("text"), str) or not block["text"].strip()
or block.get("kind") not in {"paragraph", "section_header", "title", "list_item", "table"}
or (block.get("page") is not None and (type(block["page"]) is not int or page_count is None or not 1 <= block["page"] <= page_count))):
_raise_parser_error("parser_failed")
if "\n".join(block["text"] for block in blocks) != result.get("text"):
_raise_parser_error("parser_failed")
async def _extract_upload(file: UploadFile, *, document: bool = False):
filename = file.filename or "document"
extension = "." + filename.rsplit(".", 1)[1].lower() if "." in filename else ""
data = bytearray()
@@ -1331,14 +1371,15 @@ async def extract_text(file: UploadFile = File(...)):
raise HTTPException(status_code=400, detail="The uploaded file was empty.")
if len(data) > MAX_EXTRACT_FILE_BYTES:
raise HTTPException(status_code=400, detail="The uploaded file is too large for AI extraction.")
if extension not in {".txt", ".md", ".docx", ".pdf", *IMAGE_EXTENSIONS}:
supported = {".docx", ".pdf"} if document else {".txt", ".md", ".docx", ".pdf", *IMAGE_EXTENSIONS}
if extension not in supported:
raise HTTPException(status_code=400, detail="This file type is not supported for AI extraction.")
_validate_file_signature(extension, data)
result = await asyncio.to_thread(_run_parser_child, extension, data)
result = await asyncio.to_thread(_run_parser_child, extension, data, document=True) if document else await asyncio.to_thread(_run_parser_child, extension, data)
text = result["text"]
return {
response = {
"text": text,
"ocr_used": result.get("ocr_used", False),
"content_type": file.content_type,
@@ -1346,3 +1387,17 @@ async def extract_text(file: UploadFile = File(...)):
"characters": len(text),
"file_name": filename,
}
if document:
_validate_document_result(result)
response.update({key: result[key] for key in ("blocks", "warnings", "method")})
return response
@app.post("/extract-text")
async def extract_text(file: UploadFile = File(...)):
return await _extract_upload(file)
@app.post("/cv-import/extract")
async def extract_cv_document(file: UploadFile = File(...)):
return await _extract_upload(file, document=True)