.env.example

@@ -13,6 +13,16 @@ AI_SERVICE_BASE_URL=http://ai-service:8001
 OLLAMA_BASE_URL=http://ollama:11434
 OLLAMA_MODEL=qwen2.5:7b
 
+# AI provider for the heavy /cv/* calls: ollama (default, local) | gemini | groq.
+# /summarize always stays local (distilbart). To offload a weak production GPU,
+# set AI_PROVIDER=gemini (or groq) and provide the matching key below.
+# Keys are read from the environment only — never commit real keys.
+AI_PROVIDER=ollama
+GEMINI_API_KEY=
+GEMINI_MODEL=gemini-2.0-flash
+GROQ_API_KEY=
+GROQ_MODEL=llama-3.3-70b-versatile
+
 # Optional: only needed if you want the UI to call a non-default API base URL.
 # In production the UI defaults to `/api`.
 REACT_APP_API_BASE_URL=

docker-compose.yml

@@ -74,6 +74,13 @@ services:
     environment:
       - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-http://ollama:11434}
       - OLLAMA_MODEL=${OLLAMA_MODEL:-qwen2.5:7b}
+      # AI provider for heavy /cv/* calls: ollama (default) | gemini | groq.
+      # Set AI_PROVIDER=gemini + GEMINI_API_KEY in prod to offload a weak local GPU.
+      - AI_PROVIDER=${AI_PROVIDER:-ollama}
+      - GEMINI_API_KEY=${GEMINI_API_KEY:-}
+      - GEMINI_MODEL=${GEMINI_MODEL:-gemini-2.0-flash}
+      - GROQ_API_KEY=${GROQ_API_KEY:-}
+      - GROQ_MODEL=${GROQ_MODEL:-llama-3.3-70b-versatile}
     ports:
       - "8001:8001"
     depends_on:

tools/summarizer/app.py

@@ -26,6 +26,18 @@ OCR_LANGUAGES = "eng"
 IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
 OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/")
 OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "")
+
+# AI provider router. Structured /cv/* calls (the heavy ones) dispatch through the
+# active provider so production can offload a weak local GPU to a cloud provider.
+# Default stays "ollama" so the service works keyless/local. /summarize stays local
+# (distilbart) regardless of this setting.
+AI_PROVIDER = (os.getenv("AI_PROVIDER", "ollama").strip().lower() or "ollama")
+GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "").strip()
+GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-2.0-flash").strip()
+GEMINI_BASE_URL = os.getenv("GEMINI_BASE_URL", "https://generativelanguage.googleapis.com").rstrip("/")
+GROQ_API_KEY = os.getenv("GROQ_API_KEY", "").strip()
+GROQ_MODEL = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile").strip()
+GROQ_BASE_URL = os.getenv("GROQ_BASE_URL", "https://api.groq.com/openai/v1").rstrip("/")
 SKIP_MODEL_LOAD = os.getenv("AI_SERVICE_SKIP_MODEL_LOAD", "") == "1"
 EAGER_MODEL_LOAD = os.getenv("AI_SERVICE_EAGER_MODEL_LOAD", "") == "1"
 
@@ -174,6 +186,8 @@ async def health():
         "model_disabled": MODEL_DISABLED,
         "summarize_available": MODEL_LOADED and not MODEL_DISABLED,
         "model_load_error": MODEL_LOAD_ERROR,
+        "ai_provider": AI_PROVIDER,
+        "ai_provider_configured": _provider_configured(),
         **_ollama_status(),
     }
 
@@ -390,37 +404,106 @@ def _model_summarize(text: str, max_length: int, min_length: int) -> str:
     return tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
 
 
-def _ollama_generate_json(prompt: str):
+_PROVIDER_DISPLAY = {"ollama": "Ollama", "gemini": "Gemini", "groq": "Groq"}
+
+
+def _provider_display(provider: str) -> str:
+    return _PROVIDER_DISPLAY.get(provider, provider or "AI provider")
+
+
+def _provider_configured() -> bool:
+    if AI_PROVIDER == "gemini":
+        return bool(GEMINI_API_KEY)
+    if AI_PROVIDER == "groq":
+        return bool(GROQ_API_KEY)
+    return bool(OLLAMA_MODEL)
+
+
+def _http_post_json(url: str, payload: dict, headers: dict, timeout: int) -> dict:
+    data = json.dumps(payload).encode("utf-8")
+    req = urllib_request.Request(
+        url,
+        data=data,
+        headers={"Content-Type": "application/json", **headers},
+        method="POST",
+    )
+    with urllib_request.urlopen(req, timeout=timeout) as response:
+        return json.loads(response.read().decode("utf-8"))
+
+
+def _ollama_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str:
     if not OLLAMA_MODEL:
         raise HTTPException(status_code=503, detail="OLLAMA_MODEL is not configured.")
-
+    payload = {
-    payload = json.dumps({
         "model": OLLAMA_MODEL,
         "prompt": prompt,
         "stream": False,
-        "format": "json",
+        "options": {"temperature": temperature},
-        "options": {"temperature": 0.1}
+    }
-    }).encode("utf-8")
+    if json_mode:
+        payload["format"] = "json"
+    body = _http_post_json(f"{OLLAMA_BASE_URL}/api/generate", payload, {}, timeout)
+    return (body.get("response") or "").strip()
 
-    req = urllib_request.Request(
-        f"{OLLAMA_BASE_URL}/api/generate",
-        data=payload,
-        headers={"Content-Type": "application/json"},
-        method="POST",
-    )
 
+def _gemini_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str:
+    if not GEMINI_API_KEY:
+        raise HTTPException(status_code=503, detail="GEMINI_API_KEY is not configured.")
+    generation_config = {"temperature": temperature}
+    if json_mode:
+        generation_config["responseMimeType"] = "application/json"
+    payload = {
+        "contents": [{"role": "user", "parts": [{"text": prompt}]}],
+        "generationConfig": generation_config,
+    }
+    # Pass the key via header (not the URL query string, which can leak into logs).
+    url = f"{GEMINI_BASE_URL}/v1beta/models/{GEMINI_MODEL}:generateContent"
+    body = _http_post_json(url, payload, {"x-goog-api-key": GEMINI_API_KEY}, timeout)
+    candidates = body.get("candidates") or []
+    if not candidates:
+        return ""
+    parts = (candidates[0].get("content") or {}).get("parts") or []
+    return "".join(part.get("text", "") for part in parts).strip()
+
+
+def _groq_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str:
+    if not GROQ_API_KEY:
+        raise HTTPException(status_code=503, detail="GROQ_API_KEY is not configured.")
+    payload = {
+        "model": GROQ_MODEL,
+        "messages": [{"role": "user", "content": prompt}],
+        "temperature": temperature,
+    }
+    if json_mode:
+        payload["response_format"] = {"type": "json_object"}
+    url = f"{GROQ_BASE_URL}/chat/completions"
+    body = _http_post_json(url, payload, {"Authorization": f"Bearer {GROQ_API_KEY}"}, timeout)
+    choices = body.get("choices") or []
+    if not choices:
+        return ""
+    return ((choices[0].get("message") or {}).get("content") or "").strip()
+
+
+def _provider_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str:
+    provider = AI_PROVIDER
     try:
-        with urllib_request.urlopen(req, timeout=120) as response:
+        if provider == "gemini":
-            body = json.loads(response.read().decode("utf-8"))
+            return _gemini_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout)
+        if provider == "groq":
+            return _groq_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout)
+        return _ollama_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout)
+    except HTTPException:
+        raise
     except HTTPError as ex:
-        raise HTTPException(status_code=502, detail=f"Ollama request failed with {ex.code}.")
+        raise HTTPException(status_code=502, detail=f"{_provider_display(provider)} request failed with {ex.code}.")
     except URLError as ex:
-        raise HTTPException(status_code=503, detail=f"Ollama is unreachable: {ex.reason}.")
+        raise HTTPException(status_code=503, detail=f"{_provider_display(provider)} is unreachable: {ex.reason}.")
 
-    raw = (body.get("response") or "").strip()
+
+def _ollama_generate_json(prompt: str):
+    raw = _provider_generate(prompt, json_mode=True, temperature=0.1, timeout=120)
     if not raw:
-        raise HTTPException(status_code=502, detail="Ollama returned an empty response.")
+        raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} returned an empty response.")
-
     try:
         return json.loads(raw)
     except json.JSONDecodeError:
@@ -428,39 +511,13 @@ def _ollama_generate_json(prompt: str):
         end = raw.rfind("}")
         if start >= 0 and end > start:
             return json.loads(raw[start:end + 1])
-        raise HTTPException(status_code=502, detail="Ollama did not return valid JSON.")
+        raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} did not return valid JSON.")
 
 
 def _ollama_generate_text(prompt: str) -> str:
-    if not OLLAMA_MODEL:
+    raw = _provider_generate(prompt, json_mode=False, temperature=0.2, timeout=180)
-        raise HTTPException(status_code=503, detail="OLLAMA_MODEL is not configured.")
-
-    payload = json.dumps({
-        "model": OLLAMA_MODEL,
-        "prompt": prompt,
-        "stream": False,
-        "options": {"temperature": 0.2}
-    }).encode("utf-8")
-
-    req = urllib_request.Request(
-        f"{OLLAMA_BASE_URL}/api/generate",
-        data=payload,
-        headers={"Content-Type": "application/json"},
-        method="POST",
-    )
-
-    try:
-        with urllib_request.urlopen(req, timeout=180) as response:
-            body = json.loads(response.read().decode("utf-8"))
-    except HTTPError as ex:
-        raise HTTPException(status_code=502, detail=f"Ollama request failed with {ex.code}.")
-    except URLError as ex:
-        raise HTTPException(status_code=503, detail=f"Ollama is unreachable: {ex.reason}.")
-
-    raw = (body.get("response") or "").strip()
     if not raw:
-        raise HTTPException(status_code=502, detail="Ollama returned an empty rewrite.")
+        raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} returned an empty rewrite.")
-
     return raw
 
 

tools/summarizer/tests/test_app.py

@@ -1,4 +1,5 @@
 import importlib
+import json
 import sys
 from pathlib import Path
 
@@ -141,3 +142,106 @@ def test_classify_block_defaults_missing_section_to_other(monkeypatch):
     assert payload["bullets"] == []
     assert payload["summary"] == []
     assert payload["skills"] == []
+
+
+# --- AI provider router -------------------------------------------------------
+
+class _FakeResponse:
+    def __init__(self, payload):
+        self._data = json.dumps(payload).encode("utf-8")
+
+    def read(self):
+        return self._data
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc):
+        return False
+
+
+def _install_fake_urlopen(monkeypatch, module, response_payload, captured):
+    def fake_urlopen(req, timeout=None):
+        captured["url"] = req.full_url
+        captured["headers"] = {k.lower(): v for k, v in req.header_items()}
+        captured["body"] = json.loads(req.data.decode("utf-8"))
+        return _FakeResponse(response_payload)
+
+    monkeypatch.setattr(module.urllib_request, "urlopen", fake_urlopen)
+
+
+def test_provider_defaults_to_ollama_and_is_unchanged(monkeypatch):
+    monkeypatch.delenv("AI_PROVIDER", raising=False)
+    monkeypatch.setenv("OLLAMA_BASE_URL", "http://ollama-host:11434")
+    module = load_app_module(monkeypatch, ollama_model="qwen2.5:7b")
+    assert module.AI_PROVIDER == "ollama"
+
+    captured = {}
+    _install_fake_urlopen(monkeypatch, module, {"response": '{"score": 7}'}, captured)
+
+    assert module._ollama_generate_json("hi") == {"score": 7}
+    assert captured["url"] == "http://ollama-host:11434/api/generate"
+    assert captured["body"]["model"] == "qwen2.5:7b"
+    assert captured["body"]["format"] == "json"
+    assert captured["body"]["options"]["temperature"] == 0.1
+
+
+def test_provider_gemini_dispatch(monkeypatch):
+    monkeypatch.setenv("AI_PROVIDER", "gemini")
+    monkeypatch.setenv("GEMINI_API_KEY", "test-key")
+    monkeypatch.setenv("GEMINI_MODEL", "gemini-2.0-flash")
+    module = load_app_module(monkeypatch)
+
+    captured = {}
+    payload = {"candidates": [{"content": {"parts": [{"text": '{"score": 9}'}]}}]}
+    _install_fake_urlopen(monkeypatch, module, payload, captured)
+
+    assert module._ollama_generate_json("hi") == {"score": 9}
+    assert "generativelanguage" in captured["url"]
+    assert "gemini-2.0-flash:generateContent" in captured["url"]
+    assert "key=" not in captured["url"]  # key must not be in the URL
+    assert captured["headers"].get("x-goog-api-key") == "test-key"
+    assert captured["body"]["generationConfig"]["responseMimeType"] == "application/json"
+
+
+def test_provider_groq_dispatch(monkeypatch):
+    monkeypatch.setenv("AI_PROVIDER", "groq")
+    monkeypatch.setenv("GROQ_API_KEY", "test-key")
+    module = load_app_module(monkeypatch)
+
+    captured = {}
+    payload = {"choices": [{"message": {"content": "rewritten CV text"}}]}
+    _install_fake_urlopen(monkeypatch, module, payload, captured)
+
+    assert module._ollama_generate_text("rewrite this") == "rewritten CV text"
+    assert captured["url"].endswith("/chat/completions")
+    assert captured["headers"].get("authorization") == "Bearer test-key"
+    assert captured["body"]["messages"][0]["content"] == "rewrite this"
+
+
+def test_provider_missing_cloud_key_raises_503(monkeypatch):
+    monkeypatch.setenv("AI_PROVIDER", "gemini")
+    monkeypatch.delenv("GEMINI_API_KEY", raising=False)
+    module = load_app_module(monkeypatch)
+
+    from fastapi import HTTPException
+
+    try:
+        module._ollama_generate_json("hi")
+    except HTTPException as ex:
+        assert ex.status_code == 503
+        assert "GEMINI_API_KEY" in ex.detail
+    else:
+        raise AssertionError("expected HTTPException for missing GEMINI_API_KEY")
+
+
+def test_health_reports_active_provider(monkeypatch):
+    monkeypatch.setenv("AI_PROVIDER", "gemini")
+    monkeypatch.setenv("GEMINI_API_KEY", "test-key")
+    module = load_app_module(monkeypatch)
+    client = TestClient(module.app)
+
+    payload = client.get("/health").json()
+
+    assert payload["ai_provider"] == "gemini"
+    assert payload["ai_provider_configured"] is True