using System; using System.Diagnostics; using System.Net.Http; using System.Security.Cryptography; using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; namespace JobTrackerApi.Services { public sealed record AiServiceMetrics( bool Healthy, string? Model, string? Device, bool? GpuAvailable, string? GpuName, bool? OcrAvailable, string? OcrLanguages, bool? OllamaConfigured, bool? OllamaReachable, string? OllamaModel, bool? OllamaModelAvailable, string? OllamaVersion, IReadOnlyList? OllamaInstalledModels, IReadOnlyList? OllamaLoadedModels, int? OllamaLoadedCount, double? HealthLatencyMs, double? ProbeLatencyMs, DateTimeOffset? LastProbeAt, DateTimeOffset? LastProbeSuccessAt, DateTimeOffset? LastProbeFailureAt, int ProbeFailures, int Requests, int CacheHits, int CacheMisses, int Failures, double? AverageLatencyMs, int OcrRequests, int OcrFailures, double? AverageOcrLatencyMs, DateTimeOffset? LastOcrSuccessAt, DateTimeOffset? LastOcrFailureAt, DateTimeOffset? LastSuccessAt, DateTimeOffset? LastFailureAt, string? LastError ); public sealed record AiTextExtractionResult( string? Text, bool OcrUsed, string? ContentType, int? PageCount, int Characters, string? FileName ); public interface IAiService { Task SummarizeAsync(string text, int maxLength = 150, int minLength = 30); Task SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40); Task ExtractTextAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default); Task RunProbeAsync(CancellationToken cancellationToken = default); Task GetMetricsAsync(CancellationToken cancellationToken = default); } public interface ISummarizerService : IAiService { new Task SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40); } public class SummarizerService : ISummarizerService { private const int AiSummarizeMaxInputChars = 20000; private const int AiServiceMaxSummaryLength = 256; private const int AiServiceMaxMinLength = 180; private const int AiServiceMinSummaryLength = 24; private const int AiServiceMinMinLength = 8; private readonly IHttpClientFactory _httpFactory; private readonly IMemoryCache _cache; private readonly object _metricsLock = new(); private int _requests; private int _cacheHits; private int _cacheMisses; private int _failures; private long _totalLatencyTicks; private DateTimeOffset? _lastSuccessAt; private DateTimeOffset? _lastFailureAt; private double? _lastProbeLatencyMs; private DateTimeOffset? _lastProbeAt; private DateTimeOffset? _lastProbeSuccessAt; private DateTimeOffset? _lastProbeFailureAt; private int _probeFailures; private int _ocrRequests; private int _ocrFailures; private long _totalOcrLatencyTicks; private DateTimeOffset? _lastOcrSuccessAt; private DateTimeOffset? _lastOcrFailureAt; private string? _lastError; public SummarizerService(IHttpClientFactory httpFactory, IMemoryCache cache) { _httpFactory = httpFactory; _cache = cache; } private static string BuildCacheKey(string text, int maxLength, int minLength) { var payload = $"{text}\n::{maxLength}:{minLength}"; var hash = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(payload))).ToLowerInvariant(); return $"summ:{hash}"; } private static async Task ReadErrorBodyAsync(HttpResponseMessage response, CancellationToken cancellationToken = default) { var body = await response.Content.ReadAsStringAsync(cancellationToken); if (string.IsNullOrWhiteSpace(body)) { return $"HTTP {(int)response.StatusCode}"; } try { using var doc = JsonDocument.Parse(body); if (doc.RootElement.TryGetProperty("detail", out var detailEl) && detailEl.ValueKind == JsonValueKind.String) { return $"HTTP {(int)response.StatusCode}: {detailEl.GetString()}"; } if (doc.RootElement.TryGetProperty("message", out var messageEl) && messageEl.ValueKind == JsonValueKind.String) { return $"HTTP {(int)response.StatusCode}: {messageEl.GetString()}"; } } catch (JsonException) { } body = body.Length <= 400 ? body : body[..400]; return $"HTTP {(int)response.StatusCode}: {body}"; } public async Task SummarizeAsync(string text, int maxLength = 150, int minLength = 30) { if (string.IsNullOrWhiteSpace(text)) return null; return await SummarizeCoreAsync(text, maxLength, minLength); } public Task SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40) { if (string.IsNullOrWhiteSpace(instruction) || string.IsNullOrWhiteSpace(text)) return Task.FromResult(null); return RewriteCoreAsync(instruction.Trim(), text.Trim(), maxLength, minLength); } private static string ComposeBoundedPrompt(string instruction, string text) { var prefix = $"{instruction}\n\n"; if (prefix.Length >= AiSummarizeMaxInputChars) { return prefix[..AiSummarizeMaxInputChars]; } var remaining = AiSummarizeMaxInputChars - prefix.Length; if (text.Length <= remaining) { return prefix + text; } return prefix + text[..remaining]; } private async Task RewriteCoreAsync(string instruction, string text, int maxLength, int minLength) { var normalizedMaxLength = Math.Clamp(maxLength, AiServiceMinSummaryLength, AiServiceMaxSummaryLength); var normalizedMinLength = Math.Clamp(minLength, AiServiceMinMinLength, AiServiceMaxMinLength); if (normalizedMinLength >= normalizedMaxLength) { normalizedMinLength = Math.Max(AiServiceMinMinLength, normalizedMaxLength - 1); } var composed = ComposeBoundedPrompt(instruction, text); var key = BuildCacheKey($"rewrite::{composed}", normalizedMaxLength, normalizedMinLength); Interlocked.Increment(ref _requests); if (_cache.TryGetValue(key, out var cached)) { Interlocked.Increment(ref _cacheHits); lock (_metricsLock) { _lastSuccessAt = DateTimeOffset.UtcNow; _lastError = null; } return cached; } Interlocked.Increment(ref _cacheMisses); var client = _httpFactory.CreateClient("ai-service"); var payload = JsonSerializer.Serialize(new { instruction, text, max_length = normalizedMaxLength, min_length = normalizedMinLength, }); using var content = new StringContent(payload, Encoding.UTF8, "application/json"); var sw = Stopwatch.StartNew(); try { var res = await client.PostAsync("/cv/rewrite", content); sw.Stop(); Interlocked.Add(ref _totalLatencyTicks, sw.ElapsedTicks); if (!res.IsSuccessStatusCode) { var errorBody = await ReadErrorBodyAsync(res); Interlocked.Increment(ref _failures); lock (_metricsLock) { _lastFailureAt = DateTimeOffset.UtcNow; _lastError = $"AI rewrite failed: {errorBody}"; } return null; } using var stream = await res.Content.ReadAsStreamAsync(); using var doc = await JsonDocument.ParseAsync(stream); if (doc.RootElement.TryGetProperty("rewritten_text", out var el)) { var s = el.GetString(); if (!string.IsNullOrWhiteSpace(s)) _cache.Set(key, s, TimeSpan.FromHours(6)); lock (_metricsLock) { _lastSuccessAt = DateTimeOffset.UtcNow; _lastError = null; } return s; } lock (_metricsLock) { _lastFailureAt = DateTimeOffset.UtcNow; _lastError = "AI rewrite failed: response did not contain rewritten_text."; } return null; } catch (Exception ex) { sw.Stop(); Interlocked.Add(ref _totalLatencyTicks, sw.ElapsedTicks); Interlocked.Increment(ref _failures); lock (_metricsLock) { _lastFailureAt = DateTimeOffset.UtcNow; _lastError = ex.Message; } return null; } } private async Task SummarizeCoreAsync(string text, int maxLength, int minLength) { var normalizedMaxLength = Math.Clamp(maxLength, AiServiceMinSummaryLength, AiServiceMaxSummaryLength); var normalizedMinLength = Math.Clamp(minLength, AiServiceMinMinLength, AiServiceMaxMinLength); if (normalizedMinLength >= normalizedMaxLength) { normalizedMinLength = Math.Max(AiServiceMinMinLength, normalizedMaxLength - 1); } var key = BuildCacheKey(text, normalizedMaxLength, normalizedMinLength); Interlocked.Increment(ref _requests); if (_cache.TryGetValue(key, out var cached)) { Interlocked.Increment(ref _cacheHits); lock (_metricsLock) { _lastSuccessAt = DateTimeOffset.UtcNow; _lastError = null; } return cached; } Interlocked.Increment(ref _cacheMisses); var client = _httpFactory.CreateClient("ai-service"); var payload = JsonSerializer.Serialize(new { text, max_length = normalizedMaxLength, min_length = normalizedMinLength }); using var content = new StringContent(payload, Encoding.UTF8, "application/json"); var sw = Stopwatch.StartNew(); try { var res = await client.PostAsync("/summarize", content); sw.Stop(); Interlocked.Add(ref _totalLatencyTicks, sw.ElapsedTicks); if (!res.IsSuccessStatusCode) { var errorBody = await ReadErrorBodyAsync(res); Interlocked.Increment(ref _failures); lock (_metricsLock) { _lastFailureAt = DateTimeOffset.UtcNow; _lastError = $"AI summarize failed: {errorBody}"; } return null; } using var stream = await res.Content.ReadAsStreamAsync(); using var doc = await JsonDocument.ParseAsync(stream); if (doc.RootElement.TryGetProperty("summary", out var el)) { var s = el.GetString(); if (!string.IsNullOrWhiteSpace(s)) _cache.Set(key, s, TimeSpan.FromHours(6)); lock (_metricsLock) { _lastSuccessAt = DateTimeOffset.UtcNow; _lastError = null; } return s; } return null; } catch (Exception ex) { sw.Stop(); Interlocked.Add(ref _totalLatencyTicks, sw.ElapsedTicks); Interlocked.Increment(ref _failures); lock (_metricsLock) { _lastFailureAt = DateTimeOffset.UtcNow; _lastError = ex.Message; } return null; } } public async Task ExtractTextAsync(Stream stream, string fileName, string? contentType = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(stream); if (string.IsNullOrWhiteSpace(fileName)) fileName = "document"; Interlocked.Increment(ref _ocrRequests); var client = _httpFactory.CreateClient("ai-service"); var sw = Stopwatch.StartNew(); try { using var form = new MultipartFormDataContent(); using var fileContent = new StreamContent(stream); if (!string.IsNullOrWhiteSpace(contentType)) { fileContent.Headers.ContentType = new System.Net.Http.Headers.MediaTypeHeaderValue(contentType); } form.Add(fileContent, "file", fileName); using var response = await client.PostAsync("/extract-text", form, cancellationToken); sw.Stop(); Interlocked.Add(ref _totalOcrLatencyTicks, sw.ElapsedTicks); if (!response.IsSuccessStatusCode) { var errorBody = await ReadErrorBodyAsync(response, cancellationToken); Interlocked.Increment(ref _ocrFailures); lock (_metricsLock) { _lastOcrFailureAt = DateTimeOffset.UtcNow; _lastError = $"AI extraction failed: {errorBody}"; } return null; } await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken); using var doc = await JsonDocument.ParseAsync(responseStream, cancellationToken: cancellationToken); var text = doc.RootElement.TryGetProperty("text", out var textEl) ? textEl.GetString() : null; var ocrUsed = doc.RootElement.TryGetProperty("ocr_used", out var ocrEl) && ocrEl.ValueKind is JsonValueKind.True or JsonValueKind.False && ocrEl.GetBoolean(); var detectedContentType = doc.RootElement.TryGetProperty("content_type", out var contentTypeEl) ? contentTypeEl.GetString() : contentType; int? pageCount = doc.RootElement.TryGetProperty("page_count", out var pageCountEl) && pageCountEl.ValueKind == JsonValueKind.Number ? pageCountEl.GetInt32() : null; var characters = doc.RootElement.TryGetProperty("characters", out var charactersEl) && charactersEl.ValueKind == JsonValueKind.Number ? charactersEl.GetInt32() : (text?.Length ?? 0); var returnedFileName = doc.RootElement.TryGetProperty("file_name", out var fileNameEl) ? fileNameEl.GetString() : fileName; lock (_metricsLock) { _lastOcrSuccessAt = DateTimeOffset.UtcNow; _lastError = null; } return new AiTextExtractionResult(text, ocrUsed, detectedContentType, pageCount, characters, returnedFileName); } catch (Exception ex) { sw.Stop(); Interlocked.Add(ref _totalOcrLatencyTicks, sw.ElapsedTicks); Interlocked.Increment(ref _ocrFailures); lock (_metricsLock) { _lastOcrFailureAt = DateTimeOffset.UtcNow; _lastError = ex.Message; } return null; } } public async Task RunProbeAsync(CancellationToken cancellationToken = default) { const string probeText = "AI service latency probe for Jobbjakt telemetry."; var client = _httpFactory.CreateClient("ai-service"); var payload = JsonSerializer.Serialize(new { text = probeText, max_length = 48, min_length = 12 }); using var content = new StringContent(payload, Encoding.UTF8, "application/json"); var sw = Stopwatch.StartNew(); try { using var res = await client.PostAsync("/summarize", content, cancellationToken); sw.Stop(); lock (_metricsLock) { _lastProbeAt = DateTimeOffset.UtcNow; _lastProbeLatencyMs = Math.Round(sw.Elapsed.TotalMilliseconds, 1); } if (!res.IsSuccessStatusCode) { var errorBody = await ReadErrorBodyAsync(res, cancellationToken); Interlocked.Increment(ref _probeFailures); lock (_metricsLock) { _lastProbeFailureAt = DateTimeOffset.UtcNow; _lastError = $"AI probe failed: {errorBody}"; } return; } using var stream = await res.Content.ReadAsStreamAsync(cancellationToken); using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); if (!doc.RootElement.TryGetProperty("summary", out var summaryEl) || string.IsNullOrWhiteSpace(summaryEl.GetString())) { Interlocked.Increment(ref _probeFailures); lock (_metricsLock) { _lastProbeFailureAt = DateTimeOffset.UtcNow; _lastError = "Probe returned an empty summary."; } return; } lock (_metricsLock) { _lastProbeSuccessAt = DateTimeOffset.UtcNow; _lastError = null; } } catch (Exception ex) { sw.Stop(); Interlocked.Increment(ref _probeFailures); lock (_metricsLock) { _lastProbeAt = DateTimeOffset.UtcNow; _lastProbeLatencyMs = Math.Round(sw.Elapsed.TotalMilliseconds, 1); _lastProbeFailureAt = DateTimeOffset.UtcNow; _lastError = ex.Message; } } } public async Task GetMetricsAsync(CancellationToken cancellationToken = default) { var client = _httpFactory.CreateClient("ai-service"); string? model = null; string? device = null; bool? gpuAvailable = null; string? gpuName = null; bool? ocrAvailable = null; string? ocrLanguages = null; bool? ollamaConfigured = null; bool? ollamaReachable = null; string? ollamaModel = null; bool? ollamaModelAvailable = null; string? ollamaVersion = null; List? ollamaInstalledModels = null; List? ollamaLoadedModels = null; int? ollamaLoadedCount = null; double? healthLatencyMs = null; var healthy = false; string? healthError = null; bool? summarizeAvailable = null; string? modelLoadError = null; try { var sw = Stopwatch.StartNew(); using var res = await client.GetAsync("/health", cancellationToken); sw.Stop(); healthLatencyMs = Math.Round(sw.Elapsed.TotalMilliseconds, 1); healthy = res.IsSuccessStatusCode; if (healthy) { using var stream = await res.Content.ReadAsStreamAsync(cancellationToken); using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken); if (doc.RootElement.TryGetProperty("model", out var modelEl)) model = modelEl.GetString(); if (doc.RootElement.TryGetProperty("device", out var deviceEl)) device = deviceEl.GetString(); if (doc.RootElement.TryGetProperty("gpu_available", out var gpuAvailableEl) && gpuAvailableEl.ValueKind is JsonValueKind.True or JsonValueKind.False) gpuAvailable = gpuAvailableEl.GetBoolean(); if (doc.RootElement.TryGetProperty("gpu_name", out var gpuNameEl)) gpuName = gpuNameEl.GetString(); if (doc.RootElement.TryGetProperty("ocr_available", out var ocrAvailableEl) && ocrAvailableEl.ValueKind is JsonValueKind.True or JsonValueKind.False) ocrAvailable = ocrAvailableEl.GetBoolean(); if (doc.RootElement.TryGetProperty("ocr_languages", out var ocrLanguagesEl)) ocrLanguages = ocrLanguagesEl.GetString(); if (doc.RootElement.TryGetProperty("summarize_available", out var summarizeAvailableEl) && summarizeAvailableEl.ValueKind is JsonValueKind.True or JsonValueKind.False) summarizeAvailable = summarizeAvailableEl.GetBoolean(); if (doc.RootElement.TryGetProperty("model_load_error", out var modelLoadErrorEl) && modelLoadErrorEl.ValueKind == JsonValueKind.String) modelLoadError = modelLoadErrorEl.GetString(); if (doc.RootElement.TryGetProperty("ollama_configured", out var ollamaConfiguredEl) && ollamaConfiguredEl.ValueKind is JsonValueKind.True or JsonValueKind.False) ollamaConfigured = ollamaConfiguredEl.GetBoolean(); if (doc.RootElement.TryGetProperty("ollama_reachable", out var ollamaReachableEl) && ollamaReachableEl.ValueKind is JsonValueKind.True or JsonValueKind.False) ollamaReachable = ollamaReachableEl.GetBoolean(); if (doc.RootElement.TryGetProperty("ollama_model", out var ollamaModelEl)) ollamaModel = ollamaModelEl.GetString(); if (doc.RootElement.TryGetProperty("ollama_model_available", out var ollamaModelAvailableEl) && ollamaModelAvailableEl.ValueKind is JsonValueKind.True or JsonValueKind.False) ollamaModelAvailable = ollamaModelAvailableEl.GetBoolean(); if (doc.RootElement.TryGetProperty("ollama_version", out var ollamaVersionEl)) ollamaVersion = ollamaVersionEl.GetString(); if (doc.RootElement.TryGetProperty("ollama_installed_models", out var ollamaInstalledModelsEl) && ollamaInstalledModelsEl.ValueKind == JsonValueKind.Array) { ollamaInstalledModels = ollamaInstalledModelsEl.EnumerateArray().Where(x => x.ValueKind == JsonValueKind.String).Select(x => x.GetString()).Where(x => !string.IsNullOrWhiteSpace(x)).Cast().ToList(); } if (doc.RootElement.TryGetProperty("ollama_loaded_models", out var ollamaLoadedModelsEl) && ollamaLoadedModelsEl.ValueKind == JsonValueKind.Array) { ollamaLoadedModels = ollamaLoadedModelsEl.EnumerateArray().Where(x => x.ValueKind == JsonValueKind.String).Select(x => x.GetString()).Where(x => !string.IsNullOrWhiteSpace(x)).Cast().ToList(); } if (doc.RootElement.TryGetProperty("ollama_loaded_count", out var ollamaLoadedCountEl) && ollamaLoadedCountEl.ValueKind == JsonValueKind.Number) ollamaLoadedCount = ollamaLoadedCountEl.GetInt32(); if (summarizeAvailable == false) { healthy = false; healthError = string.IsNullOrWhiteSpace(modelLoadError) ? "AI summarize capability is unavailable." : modelLoadError; } } else { healthError = $"Health check returned {(int)res.StatusCode}."; } } catch (Exception ex) { healthError = ex.Message; } var requests = Volatile.Read(ref _requests); var cacheHits = Volatile.Read(ref _cacheHits); var cacheMisses = Volatile.Read(ref _cacheMisses); var failures = Volatile.Read(ref _failures); var totalLatencyTicks = Volatile.Read(ref _totalLatencyTicks); var ocrRequests = Volatile.Read(ref _ocrRequests); var ocrFailures = Volatile.Read(ref _ocrFailures); var totalOcrLatencyTicks = Volatile.Read(ref _totalOcrLatencyTicks); DateTimeOffset? lastSuccessAt; DateTimeOffset? lastFailureAt; double? probeLatencyMs; DateTimeOffset? lastProbeAt; DateTimeOffset? lastProbeSuccessAt; DateTimeOffset? lastProbeFailureAt; DateTimeOffset? lastOcrSuccessAt; DateTimeOffset? lastOcrFailureAt; string? lastError; lock (_metricsLock) { lastSuccessAt = _lastSuccessAt; lastFailureAt = _lastFailureAt; probeLatencyMs = _lastProbeLatencyMs; lastProbeAt = _lastProbeAt; lastProbeSuccessAt = _lastProbeSuccessAt; lastProbeFailureAt = _lastProbeFailureAt; lastOcrSuccessAt = _lastOcrSuccessAt; lastOcrFailureAt = _lastOcrFailureAt; lastError = _lastError; } if (!healthy && !string.IsNullOrWhiteSpace(healthError)) { lastError = healthError; } double? averageLatencyMs = requests > 0 ? Math.Round(TimeSpan.FromTicks(totalLatencyTicks).TotalMilliseconds / requests, 1) : null; double? averageOcrLatencyMs = ocrRequests > 0 ? Math.Round(TimeSpan.FromTicks(totalOcrLatencyTicks).TotalMilliseconds / ocrRequests, 1) : null; return new AiServiceMetrics( Healthy: healthy, Model: model, Device: device, GpuAvailable: gpuAvailable, GpuName: gpuName, OcrAvailable: ocrAvailable, OcrLanguages: ocrLanguages, OllamaConfigured: ollamaConfigured, OllamaReachable: ollamaReachable, OllamaModel: ollamaModel, OllamaModelAvailable: ollamaModelAvailable, OllamaVersion: ollamaVersion, OllamaInstalledModels: ollamaInstalledModels, OllamaLoadedModels: ollamaLoadedModels, OllamaLoadedCount: ollamaLoadedCount, HealthLatencyMs: healthLatencyMs, ProbeLatencyMs: probeLatencyMs, LastProbeAt: lastProbeAt, LastProbeSuccessAt: lastProbeSuccessAt, LastProbeFailureAt: lastProbeFailureAt, ProbeFailures: Volatile.Read(ref _probeFailures), Requests: requests, CacheHits: cacheHits, CacheMisses: cacheMisses, Failures: failures, AverageLatencyMs: averageLatencyMs, OcrRequests: ocrRequests, OcrFailures: ocrFailures, AverageOcrLatencyMs: averageOcrLatencyMs, LastOcrSuccessAt: lastOcrSuccessAt, LastOcrFailureAt: lastOcrFailureAt, LastSuccessAt: lastSuccessAt, LastFailureAt: lastFailureAt, LastError: lastError ); } } public sealed class SummarizerProbeHostedService : BackgroundService { private readonly IServiceScopeFactory _scopeFactory; private readonly ILogger _logger; private readonly IConfiguration _cfg; public SummarizerProbeHostedService(IServiceScopeFactory scopeFactory, ILogger logger, IConfiguration cfg) { _scopeFactory = scopeFactory; _logger = logger; _cfg = cfg; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var enabled = _cfg.GetValue("Ai:ProbeEnabled", _cfg.GetValue("Summarizer:ProbeEnabled", true)); if (!enabled) return; var intervalSeconds = Math.Clamp(_cfg.GetValue("Ai:ProbeIntervalSeconds", _cfg.GetValue("Summarizer:ProbeIntervalSeconds", 300)), 30, 3600); var initialDelaySeconds = Math.Clamp(_cfg.GetValue("Ai:ProbeInitialDelaySeconds", _cfg.GetValue("Summarizer:ProbeInitialDelaySeconds", 15)), 0, 600); if (initialDelaySeconds > 0) { await Task.Delay(TimeSpan.FromSeconds(initialDelaySeconds), stoppingToken); } using var timer = new PeriodicTimer(TimeSpan.FromSeconds(intervalSeconds)); do { try { using var scope = _scopeFactory.CreateScope(); var aiService = scope.ServiceProvider.GetRequiredService(); await aiService.RunProbeAsync(stoppingToken); } catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) { break; } catch (Exception ex) { _logger.LogWarning(ex, "AI service latency probe failed."); } } while (await timer.WaitForNextTickAsync(stoppingToken)); } } }