feat(cv): extend templates and extraction
This commit is contained in:
+32
-4
@@ -1045,8 +1045,20 @@ async def purge_cache():
|
||||
|
||||
|
||||
def _normalize_text(value: str) -> str:
|
||||
value = value.replace("\x00", " ")
|
||||
return re.sub(r"\s+", " ", value).strip()
|
||||
"""Normalize extraction noise without destroying section, bullet, or table boundaries."""
|
||||
value = value.replace("\x00", " ").replace("\r\n", "\n").replace("\r", "\n")
|
||||
lines = []
|
||||
blank = False
|
||||
for raw_line in value.split("\n"):
|
||||
line = re.sub(r"[\t\f\v]+", " ", raw_line).strip()
|
||||
if not line:
|
||||
if lines and not blank:
|
||||
lines.append("")
|
||||
blank = True
|
||||
continue
|
||||
lines.append(line)
|
||||
blank = False
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _ocr_image(image: Image.Image) -> str:
|
||||
@@ -1063,7 +1075,10 @@ def _extract_pdf_text(data: bytes) -> tuple[str, bool, int]:
|
||||
reader = PdfReader(io.BytesIO(data))
|
||||
page_count = len(reader.pages)
|
||||
for page in reader.pages:
|
||||
extracted_pages.append(page.extract_text() or "")
|
||||
try:
|
||||
extracted_pages.append(page.extract_text(extraction_mode="layout") or "")
|
||||
except TypeError:
|
||||
extracted_pages.append(page.extract_text() or "")
|
||||
except Exception:
|
||||
extracted_pages = []
|
||||
|
||||
@@ -1084,7 +1099,20 @@ def _extract_pdf_text(data: bytes) -> tuple[str, bool, int]:
|
||||
|
||||
def _extract_docx_text(data: bytes) -> str:
|
||||
document = Document(io.BytesIO(data))
|
||||
parts = [p.text.strip() for p in document.paragraphs if p.text and p.text.strip()]
|
||||
parts = []
|
||||
blocks = document.iter_inner_content() if hasattr(document, "iter_inner_content") else document.paragraphs
|
||||
for block in blocks:
|
||||
if hasattr(block, "rows"):
|
||||
for row in block.rows:
|
||||
cells = [cell.text.strip() for cell in row.cells if cell.text and cell.text.strip()]
|
||||
if cells:
|
||||
parts.append(" | ".join(cells))
|
||||
elif getattr(block, "text", "").strip():
|
||||
text = block.text.strip()
|
||||
style = (getattr(getattr(block, "style", None), "name", "") or "").lower()
|
||||
if "role" in style and parts:
|
||||
parts.append("")
|
||||
parts.append(f"- {text}" if "bullet" in style else text)
|
||||
return _normalize_text("\n".join(parts))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user