feat(cv-import): add bounded model-free Docling extraction

tools/summarizer/parser_child.py

@@ -10,6 +10,7 @@ from pathlib import Path
 import re
 import sys
 import zipfile
+from pathlib import PurePosixPath
 
 
 MAX_EXTRACT_CHARS = 200_000
@@ -220,6 +221,143 @@ def extract(extension: str, data: bytes) -> dict[str, object]:
     raise ParserError("unsupported_type")
 
 
+def _validate_import_docx(data: bytes) -> None:
+    validate_docx_container(data)
+    from defusedxml import ElementTree
+
+    try:
+        with zipfile.ZipFile(io.BytesIO(data)) as archive:
+            names: set[str] = set()
+            for entry in archive.infolist():
+                name = entry.filename
+                path = PurePosixPath(name)
+                if name in names or path.is_absolute() or ".." in path.parts or "\\" in name or ":" in name or entry.flag_bits & 1:
+                    raise ParserError("docx_unsafe_archive")
+                names.add(name)
+                if name.lower().endswith((".bin", ".emf", ".wmf")) or name.startswith("word/embeddings/"):
+                    raise ParserError("external_content")
+                if name.endswith((".xml", ".rels")):
+                    root = ElementTree.fromstring(archive.read(entry))
+                    for node in root.iter():
+                        if node.tag.rsplit("}", 1)[-1] in {"altChunk", "OLEObject", "object"}:
+                            raise ParserError("external_content")
+                        if node.attrib.get("TargetMode", "").lower() == "external":
+                            raise ParserError("external_content")
+                        target = node.attrib.get("Target", "")
+                        if ":" in target or "\\" in target or target.startswith("/"):
+                            raise ParserError("external_content")
+    except ParserError:
+        raise
+    except Exception as exc:
+        raise ParserError("docx_signature") from exc
+
+
+def _validate_import_pdf(data: bytes) -> int:
+    if not data.startswith(b"%PDF-"):
+        raise ParserError("pdf_signature")
+    from pypdf import PdfReader
+    from pypdf.generic import ArrayObject, DictionaryObject, IndirectObject
+
+    try:
+        reader = PdfReader(io.BytesIO(data), strict=True)
+        if reader.is_encrypted:
+            raise ParserError("pdf_encrypted")
+        count = len(reader.pages)
+        if count > MAX_PDF_PAGES:
+            raise ParserError("pdf_page_limit")
+        forbidden = {"/OpenAction", "/AA", "/JS", "/JavaScript", "/Launch", "/GoToR", "/URI", "/SubmitForm", "/ImportData", "/EmbeddedFiles", "/Filespec", "/RichMedia", "/XFA"}
+        pending = [(reader.trailer, 0)]
+        seen: set[int] = set()
+        while pending:
+            value, depth = pending.pop()
+            if isinstance(value, IndirectObject):
+                value = value.get_object()
+            if not isinstance(value, (DictionaryObject, ArrayObject)) or id(value) in seen:
+                continue
+            seen.add(id(value))
+            if depth > 40 or len(seen) > 10_000:
+                raise ParserError("pdf_object_limit")
+            if isinstance(value, DictionaryObject):
+                if forbidden.intersection(value) or str(value.get("/S", "")) in forbidden or str(value.get("/Type", "")) in forbidden:
+                    raise ParserError("external_content")
+                pending.extend((item, depth + 1) for item in value.values())
+            else:
+                pending.extend((item, depth + 1) for item in value)
+        return count
+    except ParserError:
+        raise
+    except Exception as exc:
+        raise ParserError("pdf_invalid") from exc
+
+
+def extract_document(extension: str, data: bytes) -> dict[str, object]:
+    """Model-free Docling conversion of validated bytes, never URLs or local paths."""
+    if not data or len(data) > 5 * 1024 * 1024:
+        raise ParserError("file_size_limit")
+    if extension == ".docx":
+        _validate_import_docx(data)
+        page_count = None
+        warnings = ["docx_page_numbers_unavailable", "ocr_disabled"]
+    elif extension == ".pdf":
+        page_count = _validate_import_pdf(data)
+        warnings = ["pdf_native_order_no_layout_or_tables", "ocr_disabled"]
+    else:
+        raise ParserError("unsupported_type")
+
+    from docling.datamodel.base_models import DocumentStream, InputFormat, ConversionStatus
+    from docling.datamodel.backend_options import MsWordBackendOptions, ThreadedDoclingParseBackendOptions
+    from docling.datamodel.pipeline_options import ConvertPipelineOptions, NativePdfPipelineOptions
+    from docling.document_converter import DocumentConverter, WordFormatOption, NativePdfFormatOption
+    from docling_core.types.doc import TextItem, TableItem
+
+    common = dict(enable_remote_services=False, allow_external_plugins=False, do_picture_classification=False, do_picture_description=False, document_timeout=20)
+    if extension == ".pdf":
+        source_format = InputFormat.PDF
+        option = NativePdfFormatOption(
+            pipeline_options=NativePdfPipelineOptions(**common, generate_page_images=False, generate_picture_images=False, parser_threads=1),
+            backend_options=ThreadedDoclingParseBackendOptions(parser_threads=1, max_concurrent_results=1, render_pages=False, include_bitmap_images=False),
+        )
+    else:
+        source_format = InputFormat.DOCX
+        option = WordFormatOption(
+            pipeline_options=ConvertPipelineOptions(**common),
+            backend_options=MsWordBackendOptions(render_chart_images=False, max_total_bytes=MAX_DOCX_UNCOMPRESSED_BYTES, max_file_bytes=MAX_DOCX_ENTRY_BYTES, max_member_count=MAX_DOCX_ENTRIES),
+        )
+    converter = DocumentConverter(allowed_formats=[source_format], format_options={source_format: option})
+    conversion = converter.convert(DocumentStream(name="document" + extension, stream=io.BytesIO(data)), max_num_pages=MAX_PDF_PAGES, max_file_size=5 * 1024 * 1024)
+    if conversion.status != ConversionStatus.SUCCESS:
+        raise ParserError("parser_failed")
+    blocks: list[dict[str, object]] = []
+    characters = 0
+    for item, _ in conversion.document.iterate_items():
+        if isinstance(item, TableItem):
+            text = normalize_text(item.export_to_markdown(doc=conversion.document))
+            kind = "table"
+        elif isinstance(item, TextItem):
+            text = normalize_text(item.text)
+            kind = {"section_header": "section_header", "title": "title", "list_item": "list_item"}.get(item.label.value, "paragraph")
+        else:
+            continue
+        if not text:
+            continue
+        characters += len(text) + (1 if blocks else 0)
+        if characters > MAX_EXTRACT_CHARS:
+            raise ParserError("extracted_text_limit")
+        if len(blocks) >= 5000:
+            raise ParserError("block_limit")
+        page = item.prov[0].page_no if extension == ".pdf" and item.prov else None
+        blocks.append({"id": f"b{len(blocks) + 1:04d}", "text": text, "kind": kind, "page": page})
+    if not blocks:
+        raise ParserError("no_text")
+    return {"text": "\n".join(block["text"] for block in blocks), "ocr_used": False, "page_count": page_count, "blocks": blocks, "warnings": warnings, "method": "docling-native-pdf-v1" if extension == ".pdf" else "docling-docx-v1"}
+
+
+def _deny_import_side_effects(event: str, args: tuple) -> None:
+    # Model-free conversion has no reason to open a socket or invoke an external tool.
+    if event in {"socket.connect", "socket.connect_ex", "socket.getaddrinfo", "subprocess.Popen", "os.system"}:
+        raise ParserError("external_content")
+
+
 def _write_result(path: Path, result: dict[str, object]) -> None:
     path.write_text(json.dumps(result, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
 
@@ -231,12 +369,18 @@ def main() -> int:
     parser.add_argument("--extension", required=True)
     parser.add_argument("--cpu-seconds", type=int, default=20)
     parser.add_argument("--memory-bytes", type=int, default=512 * 1024 * 1024)
+    parser.add_argument("--mode", choices=("legacy", "document"), default="legacy")
     args = parser.parse_args()
     output_path = Path(args.output)
     try:
         _apply_resource_limits(args.cpu_seconds, args.memory_bytes)
         data = Path(args.input).read_bytes()
-        result = extract(args.extension.lower(), data)
+        if args.mode == "document":
+            os.environ.update({"HF_HUB_OFFLINE": "1", "HF_HUB_DISABLE_TELEMETRY": "1", "TRANSFORMERS_OFFLINE": "1"})
+            sys.addaudithook(_deny_import_side_effects)
+            result = extract_document(args.extension.lower(), data)
+        else:
+            result = extract(args.extension.lower(), data)
         if not result.get("text"):
             raise ParserError("no_text")
         _write_result(output_path, {"ok": True, **result})
@@ -244,6 +388,9 @@ def main() -> int:
     except ParserError as exc:
         _write_result(output_path, {"ok": False, "code": exc.code})
         return 2
+    except ImportError:
+        _write_result(output_path, {"ok": False, "code": "docling_unavailable" if args.mode == "document" else "parser_failed"})
+        return 3
     except BaseException:
         _write_result(output_path, {"ok": False, "code": "parser_failed"})
         return 3