cesnimda/jobtrackingapp
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(ai): enforce local-first routing

Browse Source 
Keep external providers behind server consent, task, and prompt-cost gates while persisting actual provider provenance.
This commit is contained in:
cesnimda
2026-08-09 12:30:11 +02:00
parent c3f4a57195
commit 5eb9b3cb96
29 changed files with 967 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
JobTrackerApi/Program.cs
+1
View File
@@ -38,6 +38,7 @@ else
builder.Services.AddHttpContextAccessor();
builder.Services.AddSingleton(externalOrigin);
builder.Services.AddSingleton<AiPrivacyPolicy>();
builder.Services.AddSingleton<AiOperationExecutionScope>();
builder.Services.AddTransient<AiPrivacyHeaderHandler>();
builder.Services.AddScoped<CurrentUserService>();
builder.Services.AddScoped<ICurrentUserService>(sp => sp.GetRequiredService<CurrentUserService>());
JobTrackerApi/Services/AiOperationQueue.cs
+37 -4
View File
@@ -92,7 +92,31 @@ public sealed class AiOperationAdmission(
}
public sealed record AiOperationExecutionContext(UserOperationLease Lease, string EffectivePrivacyPolicy);
public sealed record AiOperationExecutionResult(string? ResultReference);
public sealed record AiOperationExecutionResult(
string? ResultReference,
string? Provider = null,
string? Model = null,
string? RouteReason = null);
public sealed class AiOperationExecutionScope
{
private readonly AsyncLocal<AiOperationExecutionContext?> _current = new();
public AiOperationExecutionContext? Current => _current.Value;
public IDisposable Use(AiOperationExecutionContext context)
{
var previous = _current.Value;
_current.Value = context;
return new Restore(() => _current.Value = previous);
}
private sealed class Restore(Action restore) : IDisposable
{
private Action? _restore = restore;
public void Dispose() => Interlocked.Exchange(ref _restore, null)?.Invoke();
}
}
public interface IAiOperationHandler
{
@@ -110,6 +134,7 @@ public sealed class AiOperationWorker(
IServiceScopeFactory scopes,
IEnumerable<IAiOperationHandler> registeredHandlers,
AiPrivacyPolicy privacy,
AiOperationExecutionScope executionScope,
IConfiguration configuration)
{
private readonly IReadOnlyDictionary<string, IAiOperationHandler> _handlers = registeredHandlers
@@ -147,19 +172,27 @@ public sealed class AiOperationWorker(
try
{
var result = await _handlers[lease.TaskType].ExecuteAsync(
new AiOperationExecutionContext(lease, effectivePrivacy), ownerScope.ServiceProvider, execution.Token);
var context = new AiOperationExecutionContext(lease, effectivePrivacy);
using var routing = executionScope.Use(context);
var result = await _handlers[lease.TaskType].ExecuteAsync(context, ownerScope.ServiceProvider, execution.Token);
var row = await store.GetAsync(lease.OperationId, stoppingToken);
if (row?.CancellationRequestedAtUtc is not null)
await store.AcknowledgeCancellationAsync(lease.OperationId, lease.LeaseToken, stoppingToken);
else
await store.CompleteAsync(lease.OperationId, lease.LeaseToken, result.ResultReference, stoppingToken);
await store.CompleteAsync(lease.OperationId, lease.LeaseToken, result.ResultReference,
result.Provider, result.Model, result.RouteReason, stoppingToken);
}
catch (AiOperationFailure failure)
{
await store.FailAsync(lease.OperationId, lease.LeaseToken, failure.Retryable, failure.Category,
failure.Message, RetryDelay(lease.AttemptCount), stoppingToken);
}
catch (AiGenerationException failure)
{
await store.FailAsync(lease.OperationId, lease.LeaseToken, failure.Retryable, failure.Category,
failure.Message, RetryDelay(lease.AttemptCount), failure.Provider, failure.Model,
failure.RouteReason, stoppingToken);
}
catch (OperationCanceledException) when (!stoppingToken.IsCancellationRequested)
{
var row = await store.GetAsync(lease.OperationId, stoppingToken);
JobTrackerApi/Services/AiPrivacyPolicy.cs
+25 -5
View File
@@ -9,6 +9,7 @@ public sealed record AiPrivacyDecision(bool ExternalProcessingAllowed, string Pr
public sealed class AiPrivacyPolicy(IConfiguration configuration, IServiceScopeFactory scopes)
{
public const string ExternalAllowedHeader = "X-Ai-External-Allowed";
public const string TaskTypeHeader = "X-Ai-Task-Type";
public string ExternalProvider
{
@@ -19,8 +20,18 @@ public sealed class AiPrivacyPolicy(IConfiguration configuration, IServiceScopeF
}
}
public string RoutingMode
{
get
{
var mode = (configuration["Ai:RoutingMode"] ?? "local_first").Trim().ToLowerInvariant();
return mode is "local_only" or "local_first" or "external_only" ? mode : "local_only";
}
}
public bool ExternalProcessingAvailable =>
configuration.GetValue("Ai:ExternalProcessingEnabled", false)
&& RoutingMode != "local_only"
&& ExternalProvider is "gemini" or "groq";
public async Task<AiPrivacyDecision> EvaluateAsync(string? userId, CancellationToken cancellationToken = default)
@@ -44,7 +55,8 @@ public sealed class AiPrivacyPolicy(IConfiguration configuration, IServiceScopeF
public sealed class AiPrivacyHeaderHandler(
IHttpContextAccessor httpContext,
AiPrivacyPolicy privacyPolicy) : DelegatingHandler
AiPrivacyPolicy privacyPolicy,
AiOperationExecutionScope executionScope) : DelegatingHandler
{
protected override async Task<HttpResponseMessage> SendAsync(
HttpRequestMessage request,
@@ -52,10 +64,18 @@ public sealed class AiPrivacyHeaderHandler(
{
if (request.RequestUri?.AbsolutePath.StartsWith("/cv/", StringComparison.OrdinalIgnoreCase) == true)
{
var userId = httpContext.HttpContext?.User.FindFirstValue(ClaimTypes.NameIdentifier)
?? httpContext.HttpContext?.User.FindFirstValue("sub");
var decision = await privacyPolicy.EvaluateAsync(userId, cancellationToken);
if (decision.ExternalProcessingAllowed)
var operationContext = executionScope.Current;
var externalAllowed = operationContext?.EffectivePrivacyPolicy == "external_allowed";
if (operationContext is not null)
request.Headers.TryAddWithoutValidation(AiPrivacyPolicy.TaskTypeHeader, operationContext.Lease.TaskType);
else
{
var userId = httpContext.HttpContext?.User.FindFirstValue(ClaimTypes.NameIdentifier)
?? httpContext.HttpContext?.User.FindFirstValue("sub");
externalAllowed = (await privacyPolicy.EvaluateAsync(userId, cancellationToken)).ExternalProcessingAllowed;
}
if (externalAllowed)
request.Headers.TryAddWithoutValidation(AiPrivacyPolicy.ExternalAllowedHeader, "true");
}
JobTrackerApi/Services/AiWorkspaceService.cs
+23 -3
View File
@@ -134,12 +134,23 @@ public sealed class AiWorkspaceService : IAiWorkspaceService
};
var prompt = $"{instruction} {Guardrail}";
var result = await _ai.SummarizeSectionAsync(prompt, source, max, 120);
AiGenerationResult? generation;
try
{
generation = await _ai.GenerateSectionWithMetadataAsync(prompt, source, max, 120, ct);
}
catch (AiGenerationException)
{
throw new AiUnavailableException("The AI service could not generate this right now. Please try again in a moment.");
}
var result = generation?.Text;
if (string.IsNullOrWhiteSpace(result))
{
throw new AiUnavailableException("The AI service could not generate this right now. Please try again in a moment.");
}
var actualProvider = string.IsNullOrWhiteSpace(generation?.Provider) ? provider : generation.Provider;
var interaction = new AiInteraction
{
OwnerUserId = ownerUserId,
@@ -147,8 +158,17 @@ public sealed class AiWorkspaceService : IAiWorkspaceService
Module = module,
Mode = mode,
Title = title,
Provider = string.IsNullOrWhiteSpace(provider) ? "ai-service" : provider,
ResultJson = JsonSerializer.Serialize(new { text = result.Trim() }, Json),
Provider = string.IsNullOrWhiteSpace(actualProvider) ? "ai-service" : actualProvider,
ResultJson = JsonSerializer.Serialize(new
{
text = result.Trim(),
meta = new
{
model = generation?.Model,
fallbackReason = generation?.FallbackReason,
routeReason = generation?.RouteReason,
},
}, Json),
InputCharacterCount = prompt.Length + source.Length,
OutputCharacterCount = result.Trim().Length,
EstimatedTokenCount = EstimateTokens(prompt.Length + source.Length + result.Trim().Length),
JobTrackerApi/Services/SummarizerService.cs
+108 -14
View File
@@ -59,6 +59,28 @@ namespace JobTrackerApi.Services
string? FileName
);
public sealed record AiGenerationResult(
string Text,
string? Provider = null,
string? Model = null,
string? FallbackReason = null,
string? RouteReason = null);
public sealed class AiGenerationException(
string category,
string message,
bool retryable,
string? provider = null,
string? model = null,
string? routeReason = null) : Exception(message)
{
public string Category { get; } = category;
public bool Retryable { get; } = retryable;
public string? Provider { get; } = provider;
public string? Model { get; } = model;
public string? RouteReason { get; } = routeReason;
}
public interface IAiService
{
Task<string?> SummarizeAsync(string text, int maxLength = 150, int minLength = 30);
@@ -71,6 +93,17 @@ namespace JobTrackerApi.Services
public interface ISummarizerService : IAiService
{
new Task<string?> SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40);
async Task<AiGenerationResult?> GenerateSectionWithMetadataAsync(
string instruction,
string text,
int maxLength = 180,
int minLength = 40,
CancellationToken cancellationToken = default)
{
var generated = await SummarizeSectionAsync(instruction, text, maxLength, minLength);
return string.IsNullOrWhiteSpace(generated) ? null : new AiGenerationResult(generated);
}
}
public class SummarizerService : ISummarizerService
@@ -144,16 +177,40 @@ namespace JobTrackerApi.Services
return $"HTTP {(int)response.StatusCode}: {body}";
}
private static string? ReadBoundedHeader(HttpResponseMessage response, string name)
{
if (!response.Headers.TryGetValues(name, out var values)) return null;
var value = values.FirstOrDefault()?.Trim();
return string.IsNullOrWhiteSpace(value) ? null : value[..Math.Min(value.Length, 128)];
}
public async Task<string?> SummarizeAsync(string text, int maxLength = 150, int minLength = 30)
{
if (string.IsNullOrWhiteSpace(text)) return null;
return await SummarizeCoreAsync(text, maxLength, minLength);
}
public Task<string?> SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40)
public async Task<string?> SummarizeSectionAsync(string instruction, string text, int maxLength = 180, int minLength = 40)
{
if (string.IsNullOrWhiteSpace(instruction) || string.IsNullOrWhiteSpace(text)) return Task.FromResult<string?>(null);
return RewriteCoreAsync(instruction.Trim(), text.Trim(), maxLength, minLength);
try
{
return (await GenerateSectionWithMetadataAsync(instruction, text, maxLength, minLength))?.Text;
}
catch (AiGenerationException)
{
return null;
}
}
public Task<AiGenerationResult?> GenerateSectionWithMetadataAsync(
string instruction,
string text,
int maxLength = 180,
int minLength = 40,
CancellationToken cancellationToken = default)
{
if (string.IsNullOrWhiteSpace(instruction) || string.IsNullOrWhiteSpace(text)) return Task.FromResult<AiGenerationResult?>(null);
return RewriteCoreAsync(instruction.Trim(), text.Trim(), maxLength, minLength, cancellationToken);
}
private static string ComposeBoundedPrompt(string instruction, string text)
@@ -173,7 +230,12 @@ namespace JobTrackerApi.Services
return prefix + text[..remaining];
}
private async Task<string?> RewriteCoreAsync(string instruction, string text, int maxLength, int minLength)
private async Task<AiGenerationResult?> RewriteCoreAsync(
string instruction,
string text,
int maxLength,
int minLength,
CancellationToken cancellationToken)
{
var normalizedMaxLength = Math.Clamp(maxLength, AiServiceMinSummaryLength, AiServiceMaxSummaryLength);
var normalizedMinLength = Math.Clamp(minLength, AiServiceMinMinLength, AiServiceMaxMinLength);
@@ -186,7 +248,7 @@ namespace JobTrackerApi.Services
var key = BuildCacheKey($"rewrite::{composed}", normalizedMaxLength, normalizedMinLength);
Interlocked.Increment(ref _requests);
if (_cache.TryGetValue<string>(key, out var cached))
if (_cache.TryGetValue<AiGenerationResult>(key, out var cached))
{
Interlocked.Increment(ref _cacheHits);
lock (_metricsLock)
@@ -212,33 +274,48 @@ namespace JobTrackerApi.Services
try
{
var res = await client.PostAsync("/cv/rewrite", content);
using var res = await client.PostAsync("/cv/rewrite", content, cancellationToken);
sw.Stop();
Interlocked.Add(ref _totalLatencyTicks, sw.ElapsedTicks);
if (!res.IsSuccessStatusCode)
{
var errorBody = await ReadErrorBodyAsync(res);
var errorBody = await ReadErrorBodyAsync(res, cancellationToken);
Interlocked.Increment(ref _failures);
lock (_metricsLock)
{
_lastFailureAt = DateTimeOffset.UtcNow;
_lastError = $"AI rewrite failed: {errorBody}";
}
return null;
var status = (int)res.StatusCode;
throw new AiGenerationException(
status is 408 or 429 or >= 500 ? "provider_unavailable" : "provider_rejected",
"AI generation failed at the configured provider boundary.",
status is 408 or 429 or >= 500,
ReadBoundedHeader(res, "X-Ai-Provider"),
ReadBoundedHeader(res, "X-Ai-Model"),
ReadBoundedHeader(res, "X-Ai-Route-Reason"));
}
using var stream = await res.Content.ReadAsStreamAsync();
using var doc = await JsonDocument.ParseAsync(stream);
using var stream = await res.Content.ReadAsStreamAsync(cancellationToken);
using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken);
if (doc.RootElement.TryGetProperty("rewritten_text", out var el))
{
var s = el.GetString();
if (!string.IsNullOrWhiteSpace(s)) _cache.Set(key, s, TimeSpan.FromHours(6));
var result = string.IsNullOrWhiteSpace(s)
? null
: new AiGenerationResult(
s,
ReadBoundedHeader(res, "X-Ai-Provider"),
ReadBoundedHeader(res, "X-Ai-Model"),
ReadBoundedHeader(res, "X-Ai-Fallback-Reason"),
ReadBoundedHeader(res, "X-Ai-Route-Reason"));
if (result is not null) _cache.Set(key, result, TimeSpan.FromHours(6));
lock (_metricsLock)
{
_lastSuccessAt = DateTimeOffset.UtcNow;
_lastError = null;
}
return s;
return result;
}
lock (_metricsLock)
@@ -246,7 +323,21 @@ namespace JobTrackerApi.Services
_lastFailureAt = DateTimeOffset.UtcNow;
_lastError = "AI rewrite failed: response did not contain rewritten_text.";
}
return null;
throw new AiGenerationException(
"invalid_response",
"AI generation returned an invalid response.",
true,
ReadBoundedHeader(res, "X-Ai-Provider"),
ReadBoundedHeader(res, "X-Ai-Model"),
ReadBoundedHeader(res, "X-Ai-Route-Reason"));
}
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (AiGenerationException)
{
throw;
}
catch (Exception ex)
{
@@ -258,7 +349,10 @@ namespace JobTrackerApi.Services
_lastFailureAt = DateTimeOffset.UtcNow;
_lastError = ex.Message;
}
return null;
throw new AiGenerationException(
"provider_unavailable",
"AI generation could not reach the provider boundary.",
true);
}
}
JobTrackerApi/Services/UserOperationStore.cs
+39 -2
View File
@@ -159,10 +159,23 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
cancellationToken);
}
public async Task<int> CompleteAsync(Guid operationId, string leaseToken, string? resultReference, CancellationToken cancellationToken)
public Task<int> CompleteAsync(Guid operationId, string leaseToken, string? resultReference, CancellationToken cancellationToken)
=> CompleteAsync(operationId, leaseToken, resultReference, null, null, null, cancellationToken);
public async Task<int> CompleteAsync(
Guid operationId,
string leaseToken,
string? resultReference,
string? provider,
string? model,
string? completionStage,
CancellationToken cancellationToken)
{
EnsureOwnerScope();
ValidateOptional(resultReference, 256, nameof(resultReference));
ValidateOptional(provider, 128, nameof(provider));
ValidateOptional(model, 128, nameof(model));
ValidateOptional(completionStage, 64, nameof(completionStage));
var now = UtcNow;
await using var transaction = await BeginTransactionAsync(cancellationToken);
var operation = await db.UserOperations.AsNoTracking().FirstOrDefaultAsync(item => item.Id == operationId, cancellationToken);
@@ -173,6 +186,9 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
.ExecuteUpdateAsync(setters => setters
.SetProperty(operation => operation.Status, OperationStatuses.Succeeded)
.SetProperty(operation => operation.ResultReference, resultReference)
.SetProperty(operation => operation.Provider, provider)
.SetProperty(operation => operation.Model, model)
.SetProperty(operation => operation.ProgressStage, completionStage)
.SetProperty(operation => operation.CompletedAtUtc, now)
.SetProperty(operation => operation.ProgressPercent, 100)
.SetProperty(operation => operation.LeaseToken, (string?)null)
@@ -187,11 +203,27 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
return affected;
}
public async Task<bool> FailAsync(Guid operationId, string leaseToken, bool retryable, string category, string message, TimeSpan retryDelay, CancellationToken cancellationToken)
public Task<bool> FailAsync(Guid operationId, string leaseToken, bool retryable, string category, string message, TimeSpan retryDelay, CancellationToken cancellationToken)
=> FailAsync(operationId, leaseToken, retryable, category, message, retryDelay, null, null, null, cancellationToken);
public async Task<bool> FailAsync(
Guid operationId,
string leaseToken,
bool retryable,
string category,
string message,
TimeSpan retryDelay,
string? provider,
string? model,
string? progressStage,
CancellationToken cancellationToken)
{
EnsureOwnerScope();
ValidateRequired(category, 64, nameof(category));
ValidateRequired(message, 512, nameof(message));
ValidateOptional(provider, 128, nameof(provider));
ValidateOptional(model, 128, nameof(model));
ValidateOptional(progressStage, 64, nameof(progressStage));
if (retryDelay < TimeSpan.Zero || retryDelay > TimeSpan.FromHours(1)) throw new ArgumentOutOfRangeException(nameof(retryDelay));
await using var transaction = await BeginTransactionAsync(cancellationToken);
var operation = await db.UserOperations.AsNoTracking()
@@ -209,6 +241,9 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
.SetProperty(item => item.CompletedAtUtc, canRetry ? null : now)
.SetProperty(item => item.FailureCategory, category)
.SetProperty(item => item.FailureMessage, message)
.SetProperty(item => item.Provider, provider)
.SetProperty(item => item.Model, model)
.SetProperty(item => item.ProgressStage, progressStage)
.SetProperty(item => item.LeaseToken, (string?)null)
.SetProperty(item => item.LeaseExpiresAtUtc, (DateTime?)null),
cancellationToken);
@@ -290,6 +325,8 @@ public sealed class UserOperationStore(JobTrackerContext db, TimeProvider timePr
operation.FailureCategory = null;
operation.FailureMessage = null;
operation.ResultReference = null;
operation.Provider = null;
operation.Model = null;
operation.ProgressStage = null;
operation.ProgressPercent = null;
var existingNotification = await db.UserNotifications.FirstOrDefaultAsync(item => item.OperationId == operationId, cancellationToken);
JobTrackerApi/appsettings.json
+2 -1
View File
@@ -19,7 +19,8 @@
},
"Ai": {
"ExternalProcessingEnabled": false,
"ExternalProvider": "ollama"
"ExternalProvider": "ollama",
"RoutingMode": "local_first"
},
"AiQueue": {
"WorkerConcurrency": 1,