feat(ai): centralize durable usage
Add a content-free usage ledger with legacy backfill. Reserve Workspace and durable Strategy/CV work before execution so deleted history or duplicate admission cannot reset limits.
This commit is contained in:
@@ -70,6 +70,12 @@ public sealed class AccountDataExportTests
|
||||
db.GmailConnections.Add(new GmailConnection { OwnerUserId = owner.Id, GmailAddress = owner.GoogleEmail!, Scope = "mail.read", EncryptedRefreshToken = "GMAIL_REFRESH_SECRET_MUST_NOT_EXPORT", EncryptedAccessToken = "GMAIL_ACCESS_SECRET_MUST_NOT_EXPORT" });
|
||||
db.ImapConnections.Add(new ImapConnection { OwnerUserId = owner.Id, Host = "mail.example.test", Username = "owner", EncryptedPassword = "IMAP_SECRET_MUST_NOT_EXPORT" });
|
||||
db.UserOperations.Add(new UserOperation { Id = Guid.NewGuid(), OwnerUserId = owner.Id, TaskType = "synthetic", Status = OperationStatuses.Running, LeaseToken = "LEASE_SECRET_MUST_NOT_EXPORT", CreatedAtUtc = DateTime.UtcNow, AvailableAtUtc = DateTime.UtcNow });
|
||||
db.AiUsageRecords.Add(new AiUsageRecord
|
||||
{
|
||||
OwnerUserId = owner.Id, SourceType = "workspace", SourceId = "export-proof",
|
||||
TaskType = "ai.workspace", InputCharacterCount = 120, OutputCharacterCount = 40,
|
||||
EstimatedTokenCount = 40, CreatedAtUtc = DateTimeOffset.UtcNow,
|
||||
});
|
||||
db.TwoFactorRecoveryCodes.Add(new TwoFactorRecoveryCode { UserId = owner.Id, CodeHash = "RECOVERY_HASH_MUST_NOT_EXPORT", CreatedAtUtc = DateTimeOffset.UtcNow });
|
||||
await db.SaveChangesAsync();
|
||||
|
||||
@@ -102,6 +108,7 @@ public sealed class AccountDataExportTests
|
||||
.Select(ReadText));
|
||||
Assert.Contains("Synthetic Owner", readable);
|
||||
Assert.Contains("Readable application note", readable);
|
||||
Assert.Contains("export-proof", readable);
|
||||
Assert.DoesNotContain("OTHER_TENANT_PRIVATE", readable);
|
||||
Assert.DoesNotContain("PASSWORD_HASH_MUST_NOT_EXPORT", readable);
|
||||
Assert.DoesNotContain("SECURITY_STAMP_MUST_NOT_EXPORT", readable);
|
||||
|
||||
@@ -111,6 +111,17 @@ public sealed class AccountDeletionTests
|
||||
});
|
||||
var otherOperation = Operation(other.Id, OperationStatuses.Queued);
|
||||
fixture.Db.UserOperations.Add(otherOperation);
|
||||
fixture.Db.AiUsageRecords.AddRange(
|
||||
new AiUsageRecord
|
||||
{
|
||||
OwnerUserId = owner.Id, SourceType = "operation", SourceId = "owner-usage",
|
||||
TaskType = "synthetic", EstimatedTokenCount = 100, CreatedAtUtc = DateTimeOffset.UtcNow,
|
||||
},
|
||||
new AiUsageRecord
|
||||
{
|
||||
OwnerUserId = other.Id, SourceType = "operation", SourceId = "other-usage",
|
||||
TaskType = "synthetic", EstimatedTokenCount = 100, CreatedAtUtc = DateTimeOffset.UtcNow,
|
||||
});
|
||||
await fixture.Db.SaveChangesAsync();
|
||||
var generatedRoot = Path.Combine(fixture.Paths.GetOwnerCvExportsRoot(owner.Id), "20260815");
|
||||
Directory.CreateDirectory(generatedRoot);
|
||||
@@ -132,6 +143,8 @@ public sealed class AccountDeletionTests
|
||||
Assert.False(await fixture.Db.JobApplications.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == owner.Id));
|
||||
Assert.True(await fixture.Db.JobApplications.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == other.Id));
|
||||
Assert.True(await fixture.Db.UserOperations.IgnoreQueryFilters().AnyAsync(item => item.Id == otherOperation.Id));
|
||||
Assert.False(await fixture.Db.AiUsageRecords.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == owner.Id));
|
||||
Assert.True(await fixture.Db.AiUsageRecords.IgnoreQueryFilters().AnyAsync(item => item.OwnerUserId == other.Id));
|
||||
Assert.False(File.Exists(attachmentPath));
|
||||
Assert.False(File.Exists(artifactPath));
|
||||
Assert.False(File.Exists(generatedPath));
|
||||
|
||||
@@ -34,6 +34,10 @@ public sealed class AiOperationQueueTests
|
||||
Assert.Equal("local_only", first.Operation.PrivacyPolicy);
|
||||
Assert.Equal("ai_queue_full", full.Code);
|
||||
Assert.Equal(15, full.RetryAfterSeconds);
|
||||
var usage = Assert.Single(await scope.ServiceProvider.GetRequiredService<JobTrackerContext>()
|
||||
.AiUsageRecords.IgnoreQueryFilters().AsNoTracking().ToListAsync());
|
||||
Assert.Equal(first.Operation.Id.ToString("D"), usage.SourceId);
|
||||
Assert.Equal(4_000, usage.EstimatedTokenCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -137,7 +141,8 @@ public sealed class AiOperationQueueTests
|
||||
if (GenerationFailure is not null) throw GenerationFailure;
|
||||
OwnerUserId = services.GetRequiredService<ICurrentUserService>().UserId;
|
||||
PrivacyPolicy = context.EffectivePrivacyPolicy;
|
||||
return Task.FromResult(new AiOperationExecutionResult("synthetic-result", "ollama", "qwen-test", "local_primary"));
|
||||
return Task.FromResult(new AiOperationExecutionResult(
|
||||
"synthetic-result", "ollama", "qwen-test", "local_primary", 120, 40));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,6 +183,7 @@ public sealed class AiOperationQueueTests
|
||||
services.AddScoped<UserOperationStore>();
|
||||
services.AddSingleton<AiPrivacyPolicy>();
|
||||
services.AddSingleton<AiOperationExecutionScope>();
|
||||
services.AddScoped<AiUsageMeter>();
|
||||
services.AddScoped<AiOperationAdmission>();
|
||||
services.AddSingleton<IAiOperationHandler>(handler);
|
||||
services.AddSingleton<AiOperationWorker>();
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
using JobTrackerApi.Data;
|
||||
using JobTrackerApi.Models;
|
||||
using JobTrackerApi.Services;
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Moq;
|
||||
using Xunit;
|
||||
|
||||
namespace JobTrackerApi.Tests;
|
||||
|
||||
public sealed class AiUsageMeterTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Reservation_is_idempotent_and_success_can_replace_the_conservative_estimate()
|
||||
{
|
||||
await InFixtureAsync(async fixture =>
|
||||
{
|
||||
var entitlements = new AccountEntitlements(true, true, 1_000_000, 10, 10_000);
|
||||
var first = await fixture.Meter.ReserveAsync("owner", entitlements, "workspace", "same", "ai.workspace", 400, 100, default);
|
||||
var duplicate = await fixture.Meter.ReserveAsync("owner", entitlements, "workspace", "same", "ai.workspace", 800, 200, default);
|
||||
|
||||
Assert.True(first.Created);
|
||||
Assert.False(duplicate.Created);
|
||||
Assert.Equal(first.Record.Id, duplicate.Record.Id);
|
||||
await fixture.Meter.FinalizeAsync(first.Record.Id, 80, 24, default);
|
||||
|
||||
var usage = Assert.Single(await fixture.Db.AiUsageRecords.AsNoTracking().ToListAsync());
|
||||
Assert.Equal(80, usage.InputCharacterCount);
|
||||
Assert.Equal(24, usage.OutputCharacterCount);
|
||||
Assert.Equal(26, usage.EstimatedTokenCount);
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Monthly_call_and_token_limits_are_enforced_across_existing_sources()
|
||||
{
|
||||
await InFixtureAsync(async fixture =>
|
||||
{
|
||||
var tokenLimited = new AccountEntitlements(true, true, 1_000_000, 10, 500);
|
||||
await fixture.Meter.ReserveAsync("owner", tokenLimited, "operation", "one", "strategy.snapshot", 1_600, 400, default);
|
||||
var tokens = await Assert.ThrowsAsync<AiUsageLimitException>(() => fixture.Meter.ReserveAsync(
|
||||
"owner", tokenLimited, "workspace", "two", "ai.workspace", 800, 200, default));
|
||||
Assert.Equal("monthly_ai_tokens_exhausted", tokens.Code);
|
||||
|
||||
var callLimited = new AccountEntitlements(true, true, 1_000_000, 1, 10_000);
|
||||
var calls = await Assert.ThrowsAsync<AiUsageLimitException>(() => fixture.Meter.ReserveAsync(
|
||||
"owner", callLimited, "workspace", "three", "ai.workspace", 40, 10, default));
|
||||
Assert.Equal("monthly_ai_calls_exhausted", calls.Code);
|
||||
Assert.Single(await fixture.Db.AiUsageRecords.AsNoTracking().ToListAsync());
|
||||
});
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Totals_are_owner_scoped_and_survive_private_interaction_deletion()
|
||||
{
|
||||
await InFixtureAsync(async fixture =>
|
||||
{
|
||||
fixture.Db.AiUsageRecords.AddRange(
|
||||
Record("owner", "owner-source", 100),
|
||||
Record("other", "other-source", 900));
|
||||
var company = new Company { OwnerUserId = "owner", Name = "Usage test" };
|
||||
var application = new JobApplication
|
||||
{
|
||||
OwnerUserId = "owner", Company = company, JobTitle = "Usage test role",
|
||||
};
|
||||
fixture.Db.AiInteractions.Add(new AiInteraction
|
||||
{
|
||||
OwnerUserId = "owner", JobApplication = application, Module = "job-analysis",
|
||||
Title = "Private generation", Provider = "ollama",
|
||||
ResultJson = "{\"text\":\"private response\"}", CreatedAtUtc = DateTimeOffset.UtcNow,
|
||||
});
|
||||
await fixture.Db.SaveChangesAsync();
|
||||
|
||||
var before = await fixture.Meter.AllTimeAsync("owner", default);
|
||||
Assert.Equal(new AiUsageTotals(1, 300, 100, 100), before);
|
||||
|
||||
await fixture.Db.AiInteractions.ExecuteDeleteAsync();
|
||||
var after = await fixture.Meter.AllTimeAsync("owner", default);
|
||||
Assert.Equal(before, after);
|
||||
});
|
||||
}
|
||||
|
||||
private static AiUsageRecord Record(string owner, string source, int tokens) => new()
|
||||
{
|
||||
OwnerUserId = owner,
|
||||
SourceType = "workspace",
|
||||
SourceId = source,
|
||||
TaskType = "ai.workspace",
|
||||
InputCharacterCount = tokens * 3,
|
||||
OutputCharacterCount = tokens,
|
||||
EstimatedTokenCount = tokens,
|
||||
CreatedAtUtc = DateTimeOffset.UtcNow,
|
||||
};
|
||||
|
||||
private static async Task InFixtureAsync(Func<Fixture, Task> test)
|
||||
{
|
||||
await using var connection = new SqliteConnection("Data Source=:memory:");
|
||||
await connection.OpenAsync();
|
||||
var currentUser = new Mock<ICurrentUserService>();
|
||||
currentUser.SetupGet(item => item.UserId).Returns("owner");
|
||||
var options = new DbContextOptionsBuilder<JobTrackerContext>().UseSqlite(connection).Options;
|
||||
await using var db = new JobTrackerContext(options, currentUser.Object);
|
||||
await db.Database.EnsureCreatedAsync();
|
||||
await test(new Fixture(db, new AiUsageMeter(db, TimeProvider.System)));
|
||||
}
|
||||
|
||||
private sealed record Fixture(JobTrackerContext Db, AiUsageMeter Meter);
|
||||
}
|
||||
@@ -51,6 +51,9 @@ public sealed class CvProcessingOperationTests
|
||||
Assert.Equal(CvProcessingQueue.TaskType, operation.TaskType);
|
||||
Assert.Equal(CvProcessingQueue.SubjectType, operation.SubjectType);
|
||||
Assert.DoesNotContain("Ada", operation.SubjectId ?? string.Empty, StringComparison.OrdinalIgnoreCase);
|
||||
var usage = Assert.Single(await db.AiUsageRecords.IgnoreQueryFilters().ToListAsync());
|
||||
Assert.Equal(operation.Id.ToString("D"), usage.SourceId);
|
||||
Assert.Equal(AiUsageMeter.ReservationFor(CvProcessingQueue.TaskType).EstimatedTokens, usage.EstimatedTokenCount);
|
||||
}
|
||||
|
||||
Assert.True(await fixture.Provider.GetRequiredService<AiOperationWorker>().RunOnceAsync(default));
|
||||
@@ -68,6 +71,8 @@ public sealed class CvProcessingOperationTests
|
||||
Assert.Null(user.ProfileCvStructureJson);
|
||||
Assert.Null(user.CurrentCvExtractionRunId);
|
||||
Assert.Equal("operation_succeeded", (await db.UserNotifications.IgnoreQueryFilters().SingleAsync()).Kind);
|
||||
Assert.Equal(AiUsageMeter.ReservationFor(CvProcessingQueue.TaskType).EstimatedTokens,
|
||||
(await db.AiUsageRecords.IgnoreQueryFilters().SingleAsync()).EstimatedTokenCount);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -263,6 +268,7 @@ public sealed class CvProcessingOperationTests
|
||||
services.AddSingleton<AiPrivacyPolicy>();
|
||||
services.AddSingleton<AiOperationExecutionScope>();
|
||||
services.AddScoped<UserOperationStore>();
|
||||
services.AddScoped<AiUsageMeter>();
|
||||
services.AddScoped<AiOperationAdmission>();
|
||||
services.AddScoped<ICvProcessingQueue, CvProcessingQueue>();
|
||||
services.AddTransient<ProfileCvController>();
|
||||
|
||||
@@ -100,6 +100,10 @@ public sealed class SqliteDateTimeOffsetCompatibilityTests
|
||||
Interaction("user-1", job.Id, now),
|
||||
Interaction("user-1", job.Id, now.AddMonths(-2)),
|
||||
Interaction("user-2", job.Id, now));
|
||||
db.AiUsageRecords.AddRange(
|
||||
Usage("user-1", "current", now),
|
||||
Usage("user-1", "old", now.AddMonths(-2)),
|
||||
Usage("user-2", "other", now));
|
||||
db.CvExtractionRuns.AddRange(
|
||||
Run("user-1", now.AddDays(-1), "old"),
|
||||
Run("user-1", now, "new"),
|
||||
@@ -116,6 +120,7 @@ public sealed class SqliteDateTimeOffsetCompatibilityTests
|
||||
var generate = await new AiWorkspaceController(users.Object, workspaceService.Object, new ConfigurationBuilder().Build(), db)
|
||||
.Generate(job.Id, new AiWorkspaceController.GenerateRequest("job-analysis", null, null), default);
|
||||
Assert.IsType<NotFoundResult>(generate.Result);
|
||||
Assert.Equal(2, await db.AiUsageRecords.CountAsync());
|
||||
|
||||
var newestArtifactPath = Path.Combine(fixture.Paths.CvArtifactsRoot, "new.txt");
|
||||
await File.WriteAllTextAsync(newestArtifactPath, "synthetic CV");
|
||||
@@ -146,6 +151,19 @@ public sealed class SqliteDateTimeOffsetCompatibilityTests
|
||||
UpdatedAtUtc = updated,
|
||||
};
|
||||
|
||||
private static AiUsageRecord Usage(string owner, string sourceId, DateTimeOffset created) => new()
|
||||
{
|
||||
OwnerUserId = owner,
|
||||
SourceType = "test",
|
||||
SourceId = sourceId,
|
||||
TaskType = "workspace.test",
|
||||
CallCount = 1,
|
||||
InputCharacterCount = 10,
|
||||
OutputCharacterCount = 5,
|
||||
EstimatedTokenCount = 4,
|
||||
CreatedAtUtc = created,
|
||||
};
|
||||
|
||||
private static AiInteraction Interaction(string owner, int jobId, DateTimeOffset created) => new()
|
||||
{
|
||||
OwnerUserId = owner,
|
||||
|
||||
@@ -60,6 +60,12 @@ public sealed class StrategySnapshotOperationTests
|
||||
Assert.Equal("ollama", operation.Provider);
|
||||
Assert.Equal("qwen-test", operation.Model);
|
||||
Assert.Equal(1, fixture.GenerationCalls);
|
||||
var usage = Assert.Single(await db.AiUsageRecords.ToListAsync());
|
||||
Assert.Equal("strategy.snapshot", usage.TaskType);
|
||||
Assert.True(usage.InputCharacterCount > 0);
|
||||
Assert.Equal(Fixture.ValidResponse.Length, usage.OutputCharacterCount);
|
||||
Assert.Equal((usage.InputCharacterCount + usage.OutputCharacterCount + 3) / 4, usage.EstimatedTokenCount);
|
||||
Assert.True(usage.EstimatedTokenCount < AiUsageMeter.ReservationFor(StrategySnapshotService.TaskType).EstimatedTokens);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -103,7 +109,7 @@ public sealed class StrategySnapshotOperationTests
|
||||
|
||||
private sealed class Fixture : IAsyncDisposable
|
||||
{
|
||||
private const string ValidResponse = """{"strategicSummary":"Lead with delivery evidence.","cvBulletIdeas":["Built APIs"],"proofPointsToLeadWith":["Led delivery"],"coverLetterAngles":["Platform relevance"]}""";
|
||||
internal const string ValidResponse = """{"strategicSummary":"Lead with delivery evidence.","cvBulletIdeas":["Built APIs"],"proofPointsToLeadWith":["Led delivery"],"coverLetterAngles":["Platform relevance"]}""";
|
||||
private readonly SqliteConnection _connection;
|
||||
private readonly Mock<ISummarizerService> _summarizer;
|
||||
public ServiceProvider Provider { get; }
|
||||
@@ -146,6 +152,7 @@ public sealed class StrategySnapshotOperationTests
|
||||
services.AddSingleton<AiPrivacyPolicy>();
|
||||
services.AddSingleton<AiOperationExecutionScope>();
|
||||
services.AddScoped<UserOperationStore>();
|
||||
services.AddScoped<AiUsageMeter>();
|
||||
services.AddScoped<AiOperationAdmission>();
|
||||
services.AddScoped<StrategySnapshotService>();
|
||||
services.AddSingleton<IAiOperationHandler, StrategySnapshotOperationHandler>();
|
||||
|
||||
Reference in New Issue
Block a user