feat(ai): centralize durable usage
CI and Deploy / test (pull_request) Successful in 5m19s
CI and Deploy / deploy (pull_request) Has been skipped

Add a content-free usage ledger with legacy backfill. Reserve Workspace and durable Strategy/CV work before execution so deleted history or duplicate admission cannot reset limits.
This commit is contained in:
cesnimda
2026-08-15 20:03:06 +02:00
parent dbff0f8d49
commit 134aac7bcf
28 changed files with 3539 additions and 83 deletions
@@ -70,6 +70,12 @@ public sealed class AccountDataExportTests
db.GmailConnections.Add(new GmailConnection { OwnerUserId = owner.Id, GmailAddress = owner.GoogleEmail!, Scope = "mail.read", EncryptedRefreshToken = "GMAIL_REFRESH_SECRET_MUST_NOT_EXPORT", EncryptedAccessToken = "GMAIL_ACCESS_SECRET_MUST_NOT_EXPORT" });
db.ImapConnections.Add(new ImapConnection { OwnerUserId = owner.Id, Host = "mail.example.test", Username = "owner", EncryptedPassword = "IMAP_SECRET_MUST_NOT_EXPORT" });
db.UserOperations.Add(new UserOperation { Id = Guid.NewGuid(), OwnerUserId = owner.Id, TaskType = "synthetic", Status = OperationStatuses.Running, LeaseToken = "LEASE_SECRET_MUST_NOT_EXPORT", CreatedAtUtc = DateTime.UtcNow, AvailableAtUtc = DateTime.UtcNow });
db.AiUsageRecords.Add(new AiUsageRecord
{
OwnerUserId = owner.Id, SourceType = "workspace", SourceId = "export-proof",
TaskType = "ai.workspace", InputCharacterCount = 120, OutputCharacterCount = 40,
EstimatedTokenCount = 40, CreatedAtUtc = DateTimeOffset.UtcNow,
});
db.TwoFactorRecoveryCodes.Add(new TwoFactorRecoveryCode { UserId = owner.Id, CodeHash = "RECOVERY_HASH_MUST_NOT_EXPORT", CreatedAtUtc = DateTimeOffset.UtcNow });
await db.SaveChangesAsync();
@@ -102,6 +108,7 @@ public sealed class AccountDataExportTests
.Select(ReadText));
Assert.Contains("Synthetic Owner", readable);
Assert.Contains("Readable application note", readable);
Assert.Contains("export-proof", readable);
Assert.DoesNotContain("OTHER_TENANT_PRIVATE", readable);
Assert.DoesNotContain("PASSWORD_HASH_MUST_NOT_EXPORT", readable);
Assert.DoesNotContain("SECURITY_STAMP_MUST_NOT_EXPORT", readable);
@@ -111,6 +111,17 @@ public sealed class AccountDeletionTests
});
var otherOperation = Operation(other.Id, OperationStatuses.Queued);
fixture.Db.UserOperations.Add(otherOperation);
fixture.Db.AiUsageRecords.AddRange(
new AiUsageRecord
{
OwnerUserId = owner.Id, SourceType = "operation", SourceId = "owner-usage",
TaskType = "synthetic", EstimatedTokenCount = 100, CreatedAtUtc = DateTimeOffset.UtcNow,
},
new AiUsageRecord
{
OwnerUserId = other.Id, SourceType = "operation", SourceId = "other-usage",
TaskType = "synthetic", EstimatedTokenCount = 100, CreatedAtUtc = DateTimeOffset.UtcNow,
});
await fixture.Db.SaveChangesAsync();
var generatedRoot = Path.Combine(fixture.Paths.GetOwnerCvExportsRoot(owner.Id), "20260815");
Directory.CreateDirectory(generatedRoot);
@@ -132,6 +143,8 @@ public sealed class AccountDeletionTests
Assert.False(await fixture.Db.JobApplications.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == owner.Id));
Assert.True(await fixture.Db.JobApplications.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == other.Id));
Assert.True(await fixture.Db.UserOperations.IgnoreQueryFilters().AnyAsync(item => item.Id == otherOperation.Id));
Assert.False(await fixture.Db.AiUsageRecords.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == owner.Id));
Assert.True(await fixture.Db.AiUsageRecords.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == other.Id));
Assert.False(File.Exists(attachmentPath));
Assert.False(File.Exists(artifactPath));
Assert.False(File.Exists(generatedPath));
+7 -1
View File
@@ -34,6 +34,10 @@ public sealed class AiOperationQueueTests
Assert.Equal("local_only", first.Operation.PrivacyPolicy);
Assert.Equal("ai_queue_full", full.Code);
Assert.Equal(15, full.RetryAfterSeconds);
var usage = Assert.Single(await scope.ServiceProvider.GetRequiredService<JobTrackerContext>()
.AiUsageRecords.IgnoreQueryFilters().AsNoTracking().ToListAsync());
Assert.Equal(first.Operation.Id.ToString("D"), usage.SourceId);
Assert.Equal(4_000, usage.EstimatedTokenCount);
}
[Fact]
@@ -137,7 +141,8 @@ public sealed class AiOperationQueueTests
if (GenerationFailure is not null) throw GenerationFailure;
OwnerUserId = services.GetRequiredService<ICurrentUserService>().UserId;
PrivacyPolicy = context.EffectivePrivacyPolicy;
return Task.FromResult(new AiOperationExecutionResult("synthetic-result", "ollama", "qwen-test", "local_primary"));
return Task.FromResult(new AiOperationExecutionResult(
"synthetic-result", "ollama", "qwen-test", "local_primary", 120, 40));
}
}
@@ -178,6 +183,7 @@ public sealed class AiOperationQueueTests
services.AddScoped<UserOperationStore>();
services.AddSingleton<AiPrivacyPolicy>();
services.AddSingleton<AiOperationExecutionScope>();
services.AddScoped<AiUsageMeter>();
services.AddScoped<AiOperationAdmission>();
services.AddSingleton<IAiOperationHandler>(handler);
services.AddSingleton<AiOperationWorker>();
+108
View File
@@ -0,0 +1,108 @@
using JobTrackerApi.Data;
using JobTrackerApi.Models;
using JobTrackerApi.Services;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Moq;
using Xunit;
namespace JobTrackerApi.Tests;
public sealed class AiUsageMeterTests
{
[Fact]
public async Task Reservation_is_idempotent_and_success_can_replace_the_conservative_estimate()
{
await InFixtureAsync(async fixture =>
{
var entitlements = new AccountEntitlements(true, true, 1_000_000, 10, 10_000);
var first = await fixture.Meter.ReserveAsync("owner", entitlements, "workspace", "same", "ai.workspace", 400, 100, default);
var duplicate = await fixture.Meter.ReserveAsync("owner", entitlements, "workspace", "same", "ai.workspace", 800, 200, default);
Assert.True(first.Created);
Assert.False(duplicate.Created);
Assert.Equal(first.Record.Id, duplicate.Record.Id);
await fixture.Meter.FinalizeAsync(first.Record.Id, 80, 24, default);
var usage = Assert.Single(await fixture.Db.AiUsageRecords.AsNoTracking().ToListAsync());
Assert.Equal(80, usage.InputCharacterCount);
Assert.Equal(24, usage.OutputCharacterCount);
Assert.Equal(26, usage.EstimatedTokenCount);
});
}
[Fact]
public async Task Monthly_call_and_token_limits_are_enforced_across_existing_sources()
{
await InFixtureAsync(async fixture =>
{
var tokenLimited = new AccountEntitlements(true, true, 1_000_000, 10, 500);
await fixture.Meter.ReserveAsync("owner", tokenLimited, "operation", "one", "strategy.snapshot", 1_600, 400, default);
var tokens = await Assert.ThrowsAsync<AiUsageLimitException>(() => fixture.Meter.ReserveAsync(
"owner", tokenLimited, "workspace", "two", "ai.workspace", 800, 200, default));
Assert.Equal("monthly_ai_tokens_exhausted", tokens.Code);
var callLimited = new AccountEntitlements(true, true, 1_000_000, 1, 10_000);
var calls = await Assert.ThrowsAsync<AiUsageLimitException>(() => fixture.Meter.ReserveAsync(
"owner", callLimited, "workspace", "three", "ai.workspace", 40, 10, default));
Assert.Equal("monthly_ai_calls_exhausted", calls.Code);
Assert.Single(await fixture.Db.AiUsageRecords.AsNoTracking().ToListAsync());
});
}
[Fact]
public async Task Totals_are_owner_scoped_and_survive_private_interaction_deletion()
{
await InFixtureAsync(async fixture =>
{
fixture.Db.AiUsageRecords.AddRange(
Record("owner", "owner-source", 100),
Record("other", "other-source", 900));
var company = new Company { OwnerUserId = "owner", Name = "Usage test" };
var application = new JobApplication
{
OwnerUserId = "owner", Company = company, JobTitle = "Usage test role",
};
fixture.Db.AiInteractions.Add(new AiInteraction
{
OwnerUserId = "owner", JobApplication = application, Module = "job-analysis",
Title = "Private generation", Provider = "ollama",
ResultJson = "{\"text\":\"private response\"}", CreatedAtUtc = DateTimeOffset.UtcNow,
});
await fixture.Db.SaveChangesAsync();
var before = await fixture.Meter.AllTimeAsync("owner", default);
Assert.Equal(new AiUsageTotals(1, 300, 100, 100), before);
await fixture.Db.AiInteractions.ExecuteDeleteAsync();
var after = await fixture.Meter.AllTimeAsync("owner", default);
Assert.Equal(before, after);
});
}
private static AiUsageRecord Record(string owner, string source, int tokens) => new()
{
OwnerUserId = owner,
SourceType = "workspace",
SourceId = source,
TaskType = "ai.workspace",
InputCharacterCount = tokens * 3,
OutputCharacterCount = tokens,
EstimatedTokenCount = tokens,
CreatedAtUtc = DateTimeOffset.UtcNow,
};
private static async Task InFixtureAsync(Func<Fixture, Task> test)
{
await using var connection = new SqliteConnection("Data Source=:memory:");
await connection.OpenAsync();
var currentUser = new Mock<ICurrentUserService>();
currentUser.SetupGet(item => item.UserId).Returns("owner");
var options = new DbContextOptionsBuilder<JobTrackerContext>().UseSqlite(connection).Options;
await using var db = new JobTrackerContext(options, currentUser.Object);
await db.Database.EnsureCreatedAsync();
await test(new Fixture(db, new AiUsageMeter(db, TimeProvider.System)));
}
private sealed record Fixture(JobTrackerContext Db, AiUsageMeter Meter);
}
@@ -51,6 +51,9 @@ public sealed class CvProcessingOperationTests
Assert.Equal(CvProcessingQueue.TaskType, operation.TaskType);
Assert.Equal(CvProcessingQueue.SubjectType, operation.SubjectType);
Assert.DoesNotContain("Ada", operation.SubjectId ?? string.Empty, StringComparison.OrdinalIgnoreCase);
var usage = Assert.Single(await db.AiUsageRecords.IgnoreQueryFilters().ToListAsync());
Assert.Equal(operation.Id.ToString("D"), usage.SourceId);
Assert.Equal(AiUsageMeter.ReservationFor(CvProcessingQueue.TaskType).EstimatedTokens, usage.EstimatedTokenCount);
}
Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
@@ -68,6 +71,8 @@ public sealed class CvProcessingOperationTests
Assert.Null(user.ProfileCvStructureJson);
Assert.Null(user.CurrentCvExtractionRunId);
Assert.Equal("operation_succeeded", (await db.UserNotifications.IgnoreQueryFilters().SingleAsync()).Kind);
Assert.Equal(AiUsageMeter.ReservationFor(CvProcessingQueue.TaskType).EstimatedTokens,
(await db.AiUsageRecords.IgnoreQueryFilters().SingleAsync()).EstimatedTokenCount);
}
}
@@ -263,6 +268,7 @@ public sealed class CvProcessingOperationTests
services.AddSingleton<AiPrivacyPolicy>();
services.AddSingleton<AiOperationExecutionScope>();
services.AddScoped<UserOperationStore>();
services.AddScoped<AiUsageMeter>();
services.AddScoped<AiOperationAdmission>();
services.AddScoped<ICvProcessingQueue, CvProcessingQueue>();
services.AddTransient<ProfileCvController>();
@@ -100,6 +100,10 @@ public sealed class SqliteDateTimeOffsetCompatibilityTests
Interaction("user-1", job.Id, now),
Interaction("user-1", job.Id, now.AddMonths(-2)),
Interaction("user-2", job.Id, now));
db.AiUsageRecords.AddRange(
Usage("user-1", "current", now),
Usage("user-1", "old", now.AddMonths(-2)),
Usage("user-2", "other", now));
db.CvExtractionRuns.AddRange(
Run("user-1", now.AddDays(-1), "old"),
Run("user-1", now, "new"),
@@ -116,6 +120,7 @@ public sealed class SqliteDateTimeOffsetCompatibilityTests
var generate = await new AiWorkspaceController(users.Object, workspaceService.Object, new ConfigurationBuilder().Build(), db)
.Generate(job.Id, new AiWorkspaceController.GenerateRequest("job-analysis", null, null), default);
Assert.IsType<NotFoundResult>(generate.Result);
Assert.Equal(2, await db.AiUsageRecords.CountAsync());
var newestArtifactPath = Path.Combine(fixture.Paths.CvArtifactsRoot, "new.txt");
await File.WriteAllTextAsync(newestArtifactPath, "synthetic CV");
@@ -146,6 +151,19 @@ public sealed class SqliteDateTimeOffsetCompatibilityTests
UpdatedAtUtc = updated,
};
private static AiUsageRecord Usage(string owner, string sourceId, DateTimeOffset created) => new()
{
OwnerUserId = owner,
SourceType = "test",
SourceId = sourceId,
TaskType = "workspace.test",
CallCount = 1,
InputCharacterCount = 10,
OutputCharacterCount = 5,
EstimatedTokenCount = 4,
CreatedAtUtc = created,
};
private static AiInteraction Interaction(string owner, int jobId, DateTimeOffset created) => new()
{
OwnerUserId = owner,
@@ -60,6 +60,12 @@ public sealed class StrategySnapshotOperationTests
Assert.Equal("ollama", operation.Provider);
Assert.Equal("qwen-test", operation.Model);
Assert.Equal(1, fixture.GenerationCalls);
var usage = Assert.Single(await db.AiUsageRecords.ToListAsync());
Assert.Equal("strategy.snapshot", usage.TaskType);
Assert.True(usage.InputCharacterCount > 0);
Assert.Equal(Fixture.ValidResponse.Length, usage.OutputCharacterCount);
Assert.Equal((usage.InputCharacterCount + usage.OutputCharacterCount + 3) / 4, usage.EstimatedTokenCount);
Assert.True(usage.EstimatedTokenCount < AiUsageMeter.ReservationFor(StrategySnapshotService.TaskType).EstimatedTokens);
}
[Fact]
@@ -103,7 +109,7 @@ public sealed class StrategySnapshotOperationTests
private sealed class Fixture : IAsyncDisposable
{
private const string ValidResponse = """{"strategicSummary":"Lead with delivery evidence.","cvBulletIdeas":["Built APIs"],"proofPointsToLeadWith":["Led delivery"],"coverLetterAngles":["Platform relevance"]}""";
internal const string ValidResponse = """{"strategicSummary":"Lead with delivery evidence.","cvBulletIdeas":["Built APIs"],"proofPointsToLeadWith":["Led delivery"],"coverLetterAngles":["Platform relevance"]}""";
private readonly SqliteConnection _connection;
private readonly Mock<ISummarizerService> _summarizer;
public ServiceProvider Provider { get; }
@@ -146,6 +152,7 @@ public sealed class StrategySnapshotOperationTests
services.AddSingleton<AiPrivacyPolicy>();
services.AddSingleton<AiOperationExecutionScope>();
services.AddScoped<UserOperationStore>();
services.AddScoped<AiUsageMeter>();
services.AddScoped<AiOperationAdmission>();
services.AddScoped<StrategySnapshotService>();
services.AddSingleton<IAiOperationHandler, StrategySnapshotOperationHandler>();