From 824251d328f21a2f6708bff4545563544d17dcd7 Mon Sep 17 00:00:00 2001 From: cesnimda Date: Sun, 5 Jul 2026 10:40:55 +0200 Subject: [PATCH] feat(ai): provider router (ollama|gemini|groq) for heavy CV calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The structured /cv/* calls funnel through a provider router so production can offload a weak local GPU (GTX 1060) to a cloud provider without any .NET change. Default stays "ollama" (keyless/local) and /summarize remains local distilbart. - AI_PROVIDER=ollama|gemini|groq dispatch inside _ollama_generate_json/_text (entry-point names kept, so no call sites change; Ollama path is byte-identical). - Gemini (x-goog-api-key header, not URL query) and Groq (OpenAI-compatible chat/completions) added via stdlib urllib — zero new dependencies. - /health reports ai_provider + ai_provider_configured. - Keys read from env only; never logged/committed. - Compose + .env.example pass AI_PROVIDER/GEMINI_*/GROQ_* through. Tests: 11 passed (default Ollama unchanged, Gemini/Groq dispatch, missing-key 503, health reports provider). Co-Authored-By: Claude Opus 4.8 --- .env.example | 10 ++ docker-compose.yml | 7 ++ tools/summarizer/app.py | 153 ++++++++++++++++++++--------- tools/summarizer/tests/test_app.py | 104 ++++++++++++++++++++ 4 files changed, 226 insertions(+), 48 deletions(-) diff --git a/.env.example b/.env.example index 6c6c2c9..2546581 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,16 @@ AI_SERVICE_BASE_URL=http://ai-service:8001 OLLAMA_BASE_URL=http://ollama:11434 OLLAMA_MODEL=qwen2.5:7b +# AI provider for the heavy /cv/* calls: ollama (default, local) | gemini | groq. +# /summarize always stays local (distilbart). To offload a weak production GPU, +# set AI_PROVIDER=gemini (or groq) and provide the matching key below. +# Keys are read from the environment only — never commit real keys. +AI_PROVIDER=ollama +GEMINI_API_KEY= +GEMINI_MODEL=gemini-2.0-flash +GROQ_API_KEY= +GROQ_MODEL=llama-3.3-70b-versatile + # Optional: only needed if you want the UI to call a non-default API base URL. # In production the UI defaults to `/api`. REACT_APP_API_BASE_URL= diff --git a/docker-compose.yml b/docker-compose.yml index 9672408..786b55b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -74,6 +74,13 @@ services: environment: - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://ollama:11434} - OLLAMA_MODEL=${OLLAMA_MODEL:-qwen2.5:7b} + # AI provider for heavy /cv/* calls: ollama (default) | gemini | groq. + # Set AI_PROVIDER=gemini + GEMINI_API_KEY in prod to offload a weak local GPU. + - AI_PROVIDER=${AI_PROVIDER:-ollama} + - GEMINI_API_KEY=${GEMINI_API_KEY:-} + - GEMINI_MODEL=${GEMINI_MODEL:-gemini-2.0-flash} + - GROQ_API_KEY=${GROQ_API_KEY:-} + - GROQ_MODEL=${GROQ_MODEL:-llama-3.3-70b-versatile} ports: - "8001:8001" depends_on: diff --git a/tools/summarizer/app.py b/tools/summarizer/app.py index 81215a4..c0b4dfd 100644 --- a/tools/summarizer/app.py +++ b/tools/summarizer/app.py @@ -26,6 +26,18 @@ OCR_LANGUAGES = "eng" IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"} OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/") OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "") + +# AI provider router. Structured /cv/* calls (the heavy ones) dispatch through the +# active provider so production can offload a weak local GPU to a cloud provider. +# Default stays "ollama" so the service works keyless/local. /summarize stays local +# (distilbart) regardless of this setting. +AI_PROVIDER = (os.getenv("AI_PROVIDER", "ollama").strip().lower() or "ollama") +GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "").strip() +GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-2.0-flash").strip() +GEMINI_BASE_URL = os.getenv("GEMINI_BASE_URL", "https://generativelanguage.googleapis.com").rstrip("/") +GROQ_API_KEY = os.getenv("GROQ_API_KEY", "").strip() +GROQ_MODEL = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile").strip() +GROQ_BASE_URL = os.getenv("GROQ_BASE_URL", "https://api.groq.com/openai/v1").rstrip("/") SKIP_MODEL_LOAD = os.getenv("AI_SERVICE_SKIP_MODEL_LOAD", "") == "1" EAGER_MODEL_LOAD = os.getenv("AI_SERVICE_EAGER_MODEL_LOAD", "") == "1" @@ -174,6 +186,8 @@ async def health(): "model_disabled": MODEL_DISABLED, "summarize_available": MODEL_LOADED and not MODEL_DISABLED, "model_load_error": MODEL_LOAD_ERROR, + "ai_provider": AI_PROVIDER, + "ai_provider_configured": _provider_configured(), **_ollama_status(), } @@ -390,37 +404,106 @@ def _model_summarize(text: str, max_length: int, min_length: int) -> str: return tokenizer.decode(outputs[0], skip_special_tokens=True).strip() -def _ollama_generate_json(prompt: str): +_PROVIDER_DISPLAY = {"ollama": "Ollama", "gemini": "Gemini", "groq": "Groq"} + + +def _provider_display(provider: str) -> str: + return _PROVIDER_DISPLAY.get(provider, provider or "AI provider") + + +def _provider_configured() -> bool: + if AI_PROVIDER == "gemini": + return bool(GEMINI_API_KEY) + if AI_PROVIDER == "groq": + return bool(GROQ_API_KEY) + return bool(OLLAMA_MODEL) + + +def _http_post_json(url: str, payload: dict, headers: dict, timeout: int) -> dict: + data = json.dumps(payload).encode("utf-8") + req = urllib_request.Request( + url, + data=data, + headers={"Content-Type": "application/json", **headers}, + method="POST", + ) + with urllib_request.urlopen(req, timeout=timeout) as response: + return json.loads(response.read().decode("utf-8")) + + +def _ollama_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str: if not OLLAMA_MODEL: raise HTTPException(status_code=503, detail="OLLAMA_MODEL is not configured.") - - payload = json.dumps({ + payload = { "model": OLLAMA_MODEL, "prompt": prompt, "stream": False, - "format": "json", - "options": {"temperature": 0.1} - }).encode("utf-8") + "options": {"temperature": temperature}, + } + if json_mode: + payload["format"] = "json" + body = _http_post_json(f"{OLLAMA_BASE_URL}/api/generate", payload, {}, timeout) + return (body.get("response") or "").strip() - req = urllib_request.Request( - f"{OLLAMA_BASE_URL}/api/generate", - data=payload, - headers={"Content-Type": "application/json"}, - method="POST", - ) +def _gemini_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str: + if not GEMINI_API_KEY: + raise HTTPException(status_code=503, detail="GEMINI_API_KEY is not configured.") + generation_config = {"temperature": temperature} + if json_mode: + generation_config["responseMimeType"] = "application/json" + payload = { + "contents": [{"role": "user", "parts": [{"text": prompt}]}], + "generationConfig": generation_config, + } + # Pass the key via header (not the URL query string, which can leak into logs). + url = f"{GEMINI_BASE_URL}/v1beta/models/{GEMINI_MODEL}:generateContent" + body = _http_post_json(url, payload, {"x-goog-api-key": GEMINI_API_KEY}, timeout) + candidates = body.get("candidates") or [] + if not candidates: + return "" + parts = (candidates[0].get("content") or {}).get("parts") or [] + return "".join(part.get("text", "") for part in parts).strip() + + +def _groq_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str: + if not GROQ_API_KEY: + raise HTTPException(status_code=503, detail="GROQ_API_KEY is not configured.") + payload = { + "model": GROQ_MODEL, + "messages": [{"role": "user", "content": prompt}], + "temperature": temperature, + } + if json_mode: + payload["response_format"] = {"type": "json_object"} + url = f"{GROQ_BASE_URL}/chat/completions" + body = _http_post_json(url, payload, {"Authorization": f"Bearer {GROQ_API_KEY}"}, timeout) + choices = body.get("choices") or [] + if not choices: + return "" + return ((choices[0].get("message") or {}).get("content") or "").strip() + + +def _provider_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str: + provider = AI_PROVIDER try: - with urllib_request.urlopen(req, timeout=120) as response: - body = json.loads(response.read().decode("utf-8")) + if provider == "gemini": + return _gemini_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout) + if provider == "groq": + return _groq_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout) + return _ollama_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout) + except HTTPException: + raise except HTTPError as ex: - raise HTTPException(status_code=502, detail=f"Ollama request failed with {ex.code}.") + raise HTTPException(status_code=502, detail=f"{_provider_display(provider)} request failed with {ex.code}.") except URLError as ex: - raise HTTPException(status_code=503, detail=f"Ollama is unreachable: {ex.reason}.") + raise HTTPException(status_code=503, detail=f"{_provider_display(provider)} is unreachable: {ex.reason}.") - raw = (body.get("response") or "").strip() + +def _ollama_generate_json(prompt: str): + raw = _provider_generate(prompt, json_mode=True, temperature=0.1, timeout=120) if not raw: - raise HTTPException(status_code=502, detail="Ollama returned an empty response.") - + raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} returned an empty response.") try: return json.loads(raw) except json.JSONDecodeError: @@ -428,39 +511,13 @@ def _ollama_generate_json(prompt: str): end = raw.rfind("}") if start >= 0 and end > start: return json.loads(raw[start:end + 1]) - raise HTTPException(status_code=502, detail="Ollama did not return valid JSON.") + raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} did not return valid JSON.") def _ollama_generate_text(prompt: str) -> str: - if not OLLAMA_MODEL: - raise HTTPException(status_code=503, detail="OLLAMA_MODEL is not configured.") - - payload = json.dumps({ - "model": OLLAMA_MODEL, - "prompt": prompt, - "stream": False, - "options": {"temperature": 0.2} - }).encode("utf-8") - - req = urllib_request.Request( - f"{OLLAMA_BASE_URL}/api/generate", - data=payload, - headers={"Content-Type": "application/json"}, - method="POST", - ) - - try: - with urllib_request.urlopen(req, timeout=180) as response: - body = json.loads(response.read().decode("utf-8")) - except HTTPError as ex: - raise HTTPException(status_code=502, detail=f"Ollama request failed with {ex.code}.") - except URLError as ex: - raise HTTPException(status_code=503, detail=f"Ollama is unreachable: {ex.reason}.") - - raw = (body.get("response") or "").strip() + raw = _provider_generate(prompt, json_mode=False, temperature=0.2, timeout=180) if not raw: - raise HTTPException(status_code=502, detail="Ollama returned an empty rewrite.") - + raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} returned an empty rewrite.") return raw diff --git a/tools/summarizer/tests/test_app.py b/tools/summarizer/tests/test_app.py index fbfb1ae..100d3eb 100644 --- a/tools/summarizer/tests/test_app.py +++ b/tools/summarizer/tests/test_app.py @@ -1,4 +1,5 @@ import importlib +import json import sys from pathlib import Path @@ -141,3 +142,106 @@ def test_classify_block_defaults_missing_section_to_other(monkeypatch): assert payload["bullets"] == [] assert payload["summary"] == [] assert payload["skills"] == [] + + +# --- AI provider router ------------------------------------------------------- + +class _FakeResponse: + def __init__(self, payload): + self._data = json.dumps(payload).encode("utf-8") + + def read(self): + return self._data + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _install_fake_urlopen(monkeypatch, module, response_payload, captured): + def fake_urlopen(req, timeout=None): + captured["url"] = req.full_url + captured["headers"] = {k.lower(): v for k, v in req.header_items()} + captured["body"] = json.loads(req.data.decode("utf-8")) + return _FakeResponse(response_payload) + + monkeypatch.setattr(module.urllib_request, "urlopen", fake_urlopen) + + +def test_provider_defaults_to_ollama_and_is_unchanged(monkeypatch): + monkeypatch.delenv("AI_PROVIDER", raising=False) + monkeypatch.setenv("OLLAMA_BASE_URL", "http://ollama-host:11434") + module = load_app_module(monkeypatch, ollama_model="qwen2.5:7b") + assert module.AI_PROVIDER == "ollama" + + captured = {} + _install_fake_urlopen(monkeypatch, module, {"response": '{"score": 7}'}, captured) + + assert module._ollama_generate_json("hi") == {"score": 7} + assert captured["url"] == "http://ollama-host:11434/api/generate" + assert captured["body"]["model"] == "qwen2.5:7b" + assert captured["body"]["format"] == "json" + assert captured["body"]["options"]["temperature"] == 0.1 + + +def test_provider_gemini_dispatch(monkeypatch): + monkeypatch.setenv("AI_PROVIDER", "gemini") + monkeypatch.setenv("GEMINI_API_KEY", "test-key") + monkeypatch.setenv("GEMINI_MODEL", "gemini-2.0-flash") + module = load_app_module(monkeypatch) + + captured = {} + payload = {"candidates": [{"content": {"parts": [{"text": '{"score": 9}'}]}}]} + _install_fake_urlopen(monkeypatch, module, payload, captured) + + assert module._ollama_generate_json("hi") == {"score": 9} + assert "generativelanguage" in captured["url"] + assert "gemini-2.0-flash:generateContent" in captured["url"] + assert "key=" not in captured["url"] # key must not be in the URL + assert captured["headers"].get("x-goog-api-key") == "test-key" + assert captured["body"]["generationConfig"]["responseMimeType"] == "application/json" + + +def test_provider_groq_dispatch(monkeypatch): + monkeypatch.setenv("AI_PROVIDER", "groq") + monkeypatch.setenv("GROQ_API_KEY", "test-key") + module = load_app_module(monkeypatch) + + captured = {} + payload = {"choices": [{"message": {"content": "rewritten CV text"}}]} + _install_fake_urlopen(monkeypatch, module, payload, captured) + + assert module._ollama_generate_text("rewrite this") == "rewritten CV text" + assert captured["url"].endswith("/chat/completions") + assert captured["headers"].get("authorization") == "Bearer test-key" + assert captured["body"]["messages"][0]["content"] == "rewrite this" + + +def test_provider_missing_cloud_key_raises_503(monkeypatch): + monkeypatch.setenv("AI_PROVIDER", "gemini") + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + module = load_app_module(monkeypatch) + + from fastapi import HTTPException + + try: + module._ollama_generate_json("hi") + except HTTPException as ex: + assert ex.status_code == 503 + assert "GEMINI_API_KEY" in ex.detail + else: + raise AssertionError("expected HTTPException for missing GEMINI_API_KEY") + + +def test_health_reports_active_provider(monkeypatch): + monkeypatch.setenv("AI_PROVIDER", "gemini") + monkeypatch.setenv("GEMINI_API_KEY", "test-key") + module = load_app_module(monkeypatch) + client = TestClient(module.app) + + payload = client.get("/health").json() + + assert payload["ai_provider"] == "gemini" + assert payload["ai_provider_configured"] is True