cesnimda/jobtrackingapp
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(ai): provider router (ollama|gemini|groq) for heavy CV calls

Browse Source 
The structured /cv/* calls funnel through a provider router so production can
offload a weak local GPU (GTX 1060) to a cloud provider without any .NET change.
Default stays "ollama" (keyless/local) and /summarize remains local distilbart.

- AI_PROVIDER=ollama|gemini|groq dispatch inside _ollama_generate_json/_text
  (entry-point names kept, so no call sites change; Ollama path is byte-identical).
- Gemini (x-goog-api-key header, not URL query) and Groq (OpenAI-compatible
  chat/completions) added via stdlib urllib — zero new dependencies.
- /health reports ai_provider + ai_provider_configured.
- Keys read from env only; never logged/committed.
- Compose + .env.example pass AI_PROVIDER/GEMINI_*/GROQ_* through.

Tests: 11 passed (default Ollama unchanged, Gemini/Groq dispatch, missing-key 503,
health reports provider).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
cesnimda
2026-07-05 10:40:55 +02:00
parent b8ec268736
commit 824251d328
4 changed files with 226 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/summarizer/app.py
+105 -48
View File
@@ -26,6 +26,18 @@ OCR_LANGUAGES = "eng"
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp"}
OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "http://127.0.0.1:11434").rstrip("/")
OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "")
# AI provider router. Structured /cv/* calls (the heavy ones) dispatch through the
# active provider so production can offload a weak local GPU to a cloud provider.
# Default stays "ollama" so the service works keyless/local. /summarize stays local
# (distilbart) regardless of this setting.
AI_PROVIDER = (os.getenv("AI_PROVIDER", "ollama").strip().lower() or "ollama")
GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "").strip()
GEMINI_MODEL = os.getenv("GEMINI_MODEL", "gemini-2.0-flash").strip()
GEMINI_BASE_URL = os.getenv("GEMINI_BASE_URL", "https://generativelanguage.googleapis.com").rstrip("/")
GROQ_API_KEY = os.getenv("GROQ_API_KEY", "").strip()
GROQ_MODEL = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile").strip()
GROQ_BASE_URL = os.getenv("GROQ_BASE_URL", "https://api.groq.com/openai/v1").rstrip("/")
SKIP_MODEL_LOAD = os.getenv("AI_SERVICE_SKIP_MODEL_LOAD", "") == "1"
EAGER_MODEL_LOAD = os.getenv("AI_SERVICE_EAGER_MODEL_LOAD", "") == "1"
@@ -174,6 +186,8 @@ async def health():
"model_disabled": MODEL_DISABLED,
"summarize_available": MODEL_LOADED and not MODEL_DISABLED,
"model_load_error": MODEL_LOAD_ERROR,
"ai_provider": AI_PROVIDER,
"ai_provider_configured": _provider_configured(),
**_ollama_status(),
}
@@ -390,37 +404,106 @@ def _model_summarize(text: str, max_length: int, min_length: int) -> str:
return tokenizer.decode(outputs[0], skip_special_tokens=True).strip()
def _ollama_generate_json(prompt: str):
_PROVIDER_DISPLAY = {"ollama": "Ollama", "gemini": "Gemini", "groq": "Groq"}
def _provider_display(provider: str) -> str:
return _PROVIDER_DISPLAY.get(provider, provider or "AI provider")
def _provider_configured() -> bool:
if AI_PROVIDER == "gemini":
return bool(GEMINI_API_KEY)
if AI_PROVIDER == "groq":
return bool(GROQ_API_KEY)
return bool(OLLAMA_MODEL)
def _http_post_json(url: str, payload: dict, headers: dict, timeout: int) -> dict:
data = json.dumps(payload).encode("utf-8")
req = urllib_request.Request(
url,
data=data,
headers={"Content-Type": "application/json", **headers},
method="POST",
)
with urllib_request.urlopen(req, timeout=timeout) as response:
return json.loads(response.read().decode("utf-8"))
def _ollama_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str:
if not OLLAMA_MODEL:
raise HTTPException(status_code=503, detail="OLLAMA_MODEL is not configured.")
payload = json.dumps({
payload = {
"model": OLLAMA_MODEL,
"prompt": prompt,
"stream": False,
"format": "json",
"options": {"temperature": 0.1}
}).encode("utf-8")
"options": {"temperature": temperature},
}
if json_mode:
payload["format"] = "json"
body = _http_post_json(f"{OLLAMA_BASE_URL}/api/generate", payload, {}, timeout)
return (body.get("response") or "").strip()
req = urllib_request.Request(
f"{OLLAMA_BASE_URL}/api/generate",
data=payload,
headers={"Content-Type": "application/json"},
method="POST",
)
def _gemini_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str:
if not GEMINI_API_KEY:
raise HTTPException(status_code=503, detail="GEMINI_API_KEY is not configured.")
generation_config = {"temperature": temperature}
if json_mode:
generation_config["responseMimeType"] = "application/json"
payload = {
"contents": [{"role": "user", "parts": [{"text": prompt}]}],
"generationConfig": generation_config,
}
# Pass the key via header (not the URL query string, which can leak into logs).
url = f"{GEMINI_BASE_URL}/v1beta/models/{GEMINI_MODEL}:generateContent"
body = _http_post_json(url, payload, {"x-goog-api-key": GEMINI_API_KEY}, timeout)
candidates = body.get("candidates") or []
if not candidates:
return ""
parts = (candidates[0].get("content") or {}).get("parts") or []
return "".join(part.get("text", "") for part in parts).strip()
def _groq_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str:
if not GROQ_API_KEY:
raise HTTPException(status_code=503, detail="GROQ_API_KEY is not configured.")
payload = {
"model": GROQ_MODEL,
"messages": [{"role": "user", "content": prompt}],
"temperature": temperature,
}
if json_mode:
payload["response_format"] = {"type": "json_object"}
url = f"{GROQ_BASE_URL}/chat/completions"
body = _http_post_json(url, payload, {"Authorization": f"Bearer {GROQ_API_KEY}"}, timeout)
choices = body.get("choices") or []
if not choices:
return ""
return ((choices[0].get("message") or {}).get("content") or "").strip()
def _provider_generate(prompt: str, *, json_mode: bool, temperature: float, timeout: int) -> str:
provider = AI_PROVIDER
try:
with urllib_request.urlopen(req, timeout=120) as response:
body = json.loads(response.read().decode("utf-8"))
if provider == "gemini":
return _gemini_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout)
if provider == "groq":
return _groq_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout)
return _ollama_generate(prompt, json_mode=json_mode, temperature=temperature, timeout=timeout)
except HTTPException:
raise
except HTTPError as ex:
raise HTTPException(status_code=502, detail=f"Ollama request failed with {ex.code}.")
raise HTTPException(status_code=502, detail=f"{_provider_display(provider)} request failed with {ex.code}.")
except URLError as ex:
raise HTTPException(status_code=503, detail=f"Ollama is unreachable: {ex.reason}.")
raise HTTPException(status_code=503, detail=f"{_provider_display(provider)} is unreachable: {ex.reason}.")
raw = (body.get("response") or "").strip()
def _ollama_generate_json(prompt: str):
raw = _provider_generate(prompt, json_mode=True, temperature=0.1, timeout=120)
if not raw:
raise HTTPException(status_code=502, detail="Ollama returned an empty response.")
raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} returned an empty response.")
try:
return json.loads(raw)
except json.JSONDecodeError:
@@ -428,39 +511,13 @@ def _ollama_generate_json(prompt: str):
end = raw.rfind("}")
if start >= 0 and end > start:
return json.loads(raw[start:end + 1])
raise HTTPException(status_code=502, detail="Ollama did not return valid JSON.")
raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} did not return valid JSON.")
def _ollama_generate_text(prompt: str) -> str:
if not OLLAMA_MODEL:
raise HTTPException(status_code=503, detail="OLLAMA_MODEL is not configured.")
payload = json.dumps({
"model": OLLAMA_MODEL,
"prompt": prompt,
"stream": False,
"options": {"temperature": 0.2}
}).encode("utf-8")
req = urllib_request.Request(
f"{OLLAMA_BASE_URL}/api/generate",
data=payload,
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib_request.urlopen(req, timeout=180) as response:
body = json.loads(response.read().decode("utf-8"))
except HTTPError as ex:
raise HTTPException(status_code=502, detail=f"Ollama request failed with {ex.code}.")
except URLError as ex:
raise HTTPException(status_code=503, detail=f"Ollama is unreachable: {ex.reason}.")
raw = (body.get("response") or "").strip()
raw = _provider_generate(prompt, json_mode=False, temperature=0.2, timeout=180)
if not raw:
raise HTTPException(status_code=502, detail="Ollama returned an empty rewrite.")
raise HTTPException(status_code=502, detail=f"{_provider_display(AI_PROVIDER)} returned an empty rewrite.")
return raw