feat: meter AI usage
This commit is contained in:
@@ -66,6 +66,9 @@ public sealed class AiWorkspaceTests
|
||||
Assert.Equal("job-analysis", res!.Module);
|
||||
Assert.Equal("gemini", res.Provider);
|
||||
Assert.Contains("Generated suggestion", res.ResultJson);
|
||||
Assert.True(res.InputCharacterCount > 0);
|
||||
Assert.Equal("## Result\nGenerated suggestion.".Length, res.OutputCharacterCount);
|
||||
Assert.Equal((res.InputCharacterCount + res.OutputCharacterCount + 3) / 4, res.EstimatedTokenCount);
|
||||
Assert.Single(await db.AiInteractions.IgnoreQueryFilters().Where(x => x.JobApplicationId == jobId).ToListAsync());
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
using JobTrackerApi.Data;
|
||||
using JobTrackerApi.Models;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace JobTrackerApi.Controllers;
|
||||
|
||||
[ApiController]
|
||||
[Route("api/ai/usage")]
|
||||
[Authorize(AuthenticationSchemes = "local")]
|
||||
public sealed class AiUsageController : ControllerBase
|
||||
{
|
||||
private readonly UserManager<ApplicationUser> _users;
|
||||
private readonly JobTrackerContext _db;
|
||||
|
||||
public AiUsageController(UserManager<ApplicationUser> users, JobTrackerContext db)
|
||||
{
|
||||
_users = users;
|
||||
_db = db;
|
||||
}
|
||||
|
||||
public sealed record UsagePeriodDto(int Calls, long InputCharacters, long OutputCharacters, long EstimatedTokens);
|
||||
public sealed record UsageDto(UsagePeriodDto CurrentMonth, UsagePeriodDto AllTime);
|
||||
|
||||
[HttpGet]
|
||||
public async Task<ActionResult<UsageDto>> Get(CancellationToken cancellationToken)
|
||||
{
|
||||
var user = await _users.GetUserAsync(User);
|
||||
if (user is null) return Unauthorized();
|
||||
|
||||
var monthStart = new DateTimeOffset(DateTime.UtcNow.Year, DateTime.UtcNow.Month, 1, 0, 0, 0, TimeSpan.Zero);
|
||||
return Ok(new UsageDto(
|
||||
await SumAsync(_db.AiInteractions.Where(x => x.OwnerUserId == user.Id && x.CreatedAtUtc >= monthStart), cancellationToken),
|
||||
await SumAsync(_db.AiInteractions.Where(x => x.OwnerUserId == user.Id), cancellationToken)));
|
||||
}
|
||||
|
||||
private static async Task<UsagePeriodDto> SumAsync(IQueryable<AiInteraction> query, CancellationToken cancellationToken)
|
||||
{
|
||||
var totals = await query.GroupBy(_ => 1).Select(group => new UsagePeriodDto(
|
||||
group.Count(),
|
||||
group.Sum(x => (long)x.InputCharacterCount),
|
||||
group.Sum(x => (long)x.OutputCharacterCount),
|
||||
group.Sum(x => (long)x.EstimatedTokenCount))).FirstOrDefaultAsync(cancellationToken);
|
||||
return totals ?? new UsagePeriodDto(0, 0, 0, 0);
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ public sealed class AiWorkspaceController : ControllerBase
|
||||
}
|
||||
|
||||
public sealed record GenerateRequest(string Module, string? Mode, string? ExtraContext);
|
||||
public sealed record InteractionDto(int Id, string Module, string? Mode, string Title, string Provider, JsonElement Result, DateTimeOffset CreatedAtUtc);
|
||||
public sealed record InteractionDto(int Id, string Module, string? Mode, string Title, string Provider, JsonElement Result, int InputCharacterCount, int OutputCharacterCount, int EstimatedTokenCount, DateTimeOffset CreatedAtUtc);
|
||||
|
||||
[HttpGet("modules")]
|
||||
public ActionResult<object> Modules() => Ok(new { modules = _workspace.Modules, provider = ResolveProvider() });
|
||||
@@ -86,5 +86,5 @@ public sealed class AiWorkspaceController : ControllerBase
|
||||
private static InteractionDto ToDto(AiInteraction x) => new(
|
||||
x.Id, x.Module, x.Mode, x.Title, x.Provider,
|
||||
JsonSerializer.Deserialize<JsonElement>(string.IsNullOrWhiteSpace(x.ResultJson) ? "{}" : x.ResultJson),
|
||||
x.CreatedAtUtc);
|
||||
x.InputCharacterCount, x.OutputCharacterCount, x.EstimatedTokenCount, x.CreatedAtUtc);
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace JobTrackerApi.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddAiUsageMetering : Migration
|
||||
{
|
||||
// Intentionally a no-op: StartupInitializationExtensions owns idempotent SQLite/MariaDB
|
||||
// column reconciliation and runs before EF migrations. The snapshot records the model change;
|
||||
// the reconciler performs the provider-safe DDL without duplicate-column failures.
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,12 @@ namespace JobTrackerApi.Migrations
|
||||
b.Property<DateTimeOffset>("CreatedAtUtc")
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int>("EstimatedTokenCount")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
b.Property<int>("InputCharacterCount")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
b.Property<int>("JobApplicationId")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
@@ -37,6 +43,9 @@ namespace JobTrackerApi.Migrations
|
||||
.HasMaxLength(64)
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
b.Property<int>("OutputCharacterCount")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
b.Property<string>("OwnerUserId")
|
||||
.IsRequired()
|
||||
.HasMaxLength(255)
|
||||
|
||||
@@ -133,7 +133,8 @@ public sealed class AiWorkspaceService : IAiWorkspaceService
|
||||
_ => throw new ArgumentException($"Unknown AI module '{module}'."),
|
||||
};
|
||||
|
||||
var result = await _ai.SummarizeSectionAsync($"{instruction} {Guardrail}", source, max, 120);
|
||||
var prompt = $"{instruction} {Guardrail}";
|
||||
var result = await _ai.SummarizeSectionAsync(prompt, source, max, 120);
|
||||
if (string.IsNullOrWhiteSpace(result))
|
||||
{
|
||||
throw new AiUnavailableException("The AI service could not generate this right now. Please try again in a moment.");
|
||||
@@ -148,6 +149,9 @@ public sealed class AiWorkspaceService : IAiWorkspaceService
|
||||
Title = title,
|
||||
Provider = string.IsNullOrWhiteSpace(provider) ? "ai-service" : provider,
|
||||
ResultJson = JsonSerializer.Serialize(new { text = result.Trim() }, Json),
|
||||
InputCharacterCount = prompt.Length + source.Length,
|
||||
OutputCharacterCount = result.Trim().Length,
|
||||
EstimatedTokenCount = EstimateTokens(prompt.Length + source.Length + result.Trim().Length),
|
||||
CreatedAtUtc = DateTimeOffset.UtcNow,
|
||||
};
|
||||
_db.AiInteractions.Add(interaction);
|
||||
@@ -174,6 +178,8 @@ public sealed class AiWorkspaceService : IAiWorkspaceService
|
||||
return true;
|
||||
}
|
||||
|
||||
internal static int EstimateTokens(int characterCount) => Math.Max(0, (characterCount + 3) / 4);
|
||||
|
||||
private static string? NormalizeMode(string module, string? mode)
|
||||
{
|
||||
if (module != "cover-letter") return null;
|
||||
|
||||
@@ -1016,10 +1016,16 @@ public static class StartupInitializationExtensions
|
||||
"Title" TEXT NOT NULL,
|
||||
"Provider" TEXT NOT NULL,
|
||||
"ResultJson" TEXT NOT NULL,
|
||||
"InputCharacterCount" INTEGER NOT NULL DEFAULT 0,
|
||||
"OutputCharacterCount" INTEGER NOT NULL DEFAULT 0,
|
||||
"EstimatedTokenCount" INTEGER NOT NULL DEFAULT 0,
|
||||
"CreatedAtUtc" TEXT NOT NULL,
|
||||
CONSTRAINT "FK_AiInteractions_JobApplications_JobApplicationId" FOREIGN KEY ("JobApplicationId") REFERENCES "JobApplications" ("Id") ON DELETE CASCADE
|
||||
);
|
||||
""");
|
||||
EnsureColumn(c, "AiInteractions", "InputCharacterCount", "ALTER TABLE AiInteractions ADD COLUMN InputCharacterCount INTEGER NOT NULL DEFAULT 0;");
|
||||
EnsureColumn(c, "AiInteractions", "OutputCharacterCount", "ALTER TABLE AiInteractions ADD COLUMN OutputCharacterCount INTEGER NOT NULL DEFAULT 0;");
|
||||
EnsureColumn(c, "AiInteractions", "EstimatedTokenCount", "ALTER TABLE AiInteractions ADD COLUMN EstimatedTokenCount INTEGER NOT NULL DEFAULT 0;");
|
||||
Exec(c, """CREATE INDEX IF NOT EXISTS "IX_AiInteractions_JobApplicationId" ON "AiInteractions" ("JobApplicationId");""");
|
||||
Exec(c, """CREATE INDEX IF NOT EXISTS "IX_AiInteractions_Owner_Job_Module_Created" ON "AiInteractions" ("OwnerUserId", "JobApplicationId", "Module", "CreatedAtUtc");""");
|
||||
}
|
||||
@@ -1760,6 +1766,9 @@ public static class StartupInitializationExtensions
|
||||
`Title` varchar(255) NOT NULL,
|
||||
`Provider` varchar(100) NOT NULL,
|
||||
`ResultJson` longtext NOT NULL,
|
||||
`InputCharacterCount` int NOT NULL DEFAULT 0,
|
||||
`OutputCharacterCount` int NOT NULL DEFAULT 0,
|
||||
`EstimatedTokenCount` int NOT NULL DEFAULT 0,
|
||||
`CreatedAtUtc` datetime(6) NOT NULL,
|
||||
PRIMARY KEY (`Id`),
|
||||
CONSTRAINT `FK_AiInteractions_JobApplications_JobApplicationId` FOREIGN KEY (`JobApplicationId`) REFERENCES `JobApplications` (`Id`) ON DELETE CASCADE
|
||||
@@ -1842,6 +1851,9 @@ public static class StartupInitializationExtensions
|
||||
EnsureMySqlAutoIncrementPrimaryKey(conn, "CvVariants", "Id");
|
||||
EnsureMySqlAutoIncrementPrimaryKey(conn, "CvVariantVersions", "Id");
|
||||
EnsureMySqlAutoIncrementPrimaryKey(conn, "AiInteractions", "Id");
|
||||
EnsureMySqlColumn(conn, "AiInteractions", "InputCharacterCount", "ALTER TABLE `AiInteractions` ADD COLUMN `InputCharacterCount` int NOT NULL DEFAULT 0;");
|
||||
EnsureMySqlColumn(conn, "AiInteractions", "OutputCharacterCount", "ALTER TABLE `AiInteractions` ADD COLUMN `OutputCharacterCount` int NOT NULL DEFAULT 0;");
|
||||
EnsureMySqlColumn(conn, "AiInteractions", "EstimatedTokenCount", "ALTER TABLE `AiInteractions` ADD COLUMN `EstimatedTokenCount` int NOT NULL DEFAULT 0;");
|
||||
|
||||
foreach (var (ixTable, ixName, ixColumns, ixUnique) in new[]
|
||||
{
|
||||
|
||||
@@ -28,5 +28,10 @@ public sealed class AiInteraction
|
||||
// meta carries any structured extras (e.g. career-match percent).
|
||||
public string ResultJson { get; set; } = string.Empty;
|
||||
|
||||
// Provider-neutral usage meter. Token count is estimated because the sidecar currently returns text only.
|
||||
public int InputCharacterCount { get; set; }
|
||||
public int OutputCharacterCount { get; set; }
|
||||
public int EstimatedTokenCount { get; set; }
|
||||
|
||||
public DateTimeOffset CreatedAtUtc { get; set; } = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
@@ -55,10 +55,13 @@ master profile text ────────────┘ │
|
||||
overwritten. This is deliberately distinct from `AiWorkspaceNote` (a one-row-per-type *cache* for
|
||||
candidate-fit/focus-plan). History gives the user restore/reuse (re-surface a past result), compare
|
||||
(view two side by side), copy, and delete. `ResultJson` is `{ text, meta? }`; `Provider` records which
|
||||
provider produced it. Tenant-scoped (owner query filter), cascades with the application.
|
||||
provider produced it. Each row also stores input/output character counts and a conservative estimated
|
||||
token count (characters ÷ 4); the estimate is provider-neutral because the sidecar currently returns
|
||||
text rather than provider billing metadata. Tenant-scoped (owner query filter), cascades with the application.
|
||||
|
||||
API (`AiWorkspaceController`, `/api/jobapplications/{id}/ai`): `GET modules` (+ current provider),
|
||||
`POST generate`, `GET history?module=`, `DELETE history/{id}`.
|
||||
`POST generate`, `GET history?module=`, `DELETE history/{id}`. `GET /api/ai/usage` returns current-month
|
||||
and all-time totals; the workspace displays the monthly calls and estimated tokens.
|
||||
|
||||
## Provider abstraction
|
||||
|
||||
|
||||
@@ -391,7 +391,7 @@ No structured sink (Seq/OTLP), no in-app log rotation, no ProblemDetails standar
|
||||
|---|---|---|
|
||||
| Medium | DataProtection keys recoverable from git history (`519c32e`, `955cae6`) | **Open — rotation required, needs an operator** |
|
||||
| Medium | **CORS: `Cors:Origins="*"` triggers `SetIsOriginAllowed(_ => true)` + `AllowCredentials()`** (`Program.cs:96-102`) — reflected-origin with cookies = session theft from any site. Not currently active (compose never sets `Cors__Origins`, so it defaults to `localhost:3000`), but it is one config value away. | **Open — landmine** |
|
||||
| Medium | No AI cost ceiling (no quota, no metering, unthrottled) | Open |
|
||||
| Medium | AI cost ceiling | Metering shipped in Phase 5; enforce quotas before open registration (Phase 7). |
|
||||
| Low | No CAPTCHA (rate limiting only) | Open — blocks public signup |
|
||||
| Low | Unbounded storage: attachments, CV artifacts, extraction runs, base64 avatars | Open |
|
||||
| Low | Backup / DPAPI is Windows-oriented — verify behaviour on Linux prod | Unverified |
|
||||
|
||||
@@ -146,14 +146,14 @@ Goal: polish. This is the healthiest area — grounding in the structured profil
|
||||
> `ISummarizerService`/ai-service provider abstraction and stored as **append-only history**
|
||||
> (`AiInteraction`) with reuse / compare / copy / delete. Suggestion-only throughout; nothing
|
||||
> auto-applies; `Markdown` renders React nodes (no HTML-injection surface). See
|
||||
> `docs/architecture/ai-career-assistant.md`. **Still open:** per-request user-selectable providers
|
||||
> (needs an ai-service per-request override + a configured key per provider — deployment/credential
|
||||
> work, documented as an extension point); 5.2 AI usage metering (Phase 7 blocker) remains.
|
||||
> `docs/architecture/ai-career-assistant.md`. **Phase 5 completed 2026-07-30:** provider-neutral usage
|
||||
> metering and visible monthly totals now close the Phase 7 cost-control prerequisite. Per-request
|
||||
> provider choice remains deliberately deferred by ADR-004.
|
||||
|
||||
| # | Task | Priority | Difficulty | Dependencies | Expected value |
|
||||
|---|---|---|---|---|---|
|
||||
| 5.1 | ✅ **DONE (2026-07-30)** — fixed `docs/00-ai-context.md` to match the code. **Decided 2026-07-17: do NOT build the abstraction.** | **P1** | **S** | none | The doc describes a provider interface over OpenAI/Gemini/Claude/Ollama with admin control and per-user choice. Reality: one `AI_PROVIDER` env var over Ollama/Gemini/Groq. Multi-provider cloud AI also undermines the privacy moat (see `docs/research/competitors.md` §4). Revisit only if a customer asks. `docs/architecture/current.md` §9 already records the truth. |
|
||||
| 5.2 | **AI usage metering** | **P1** | **M** | 1.5 | No quota, no tracking, no ceiling. Hard blocker for Phase 7; a cost risk today with `AI_PROVIDER=gemini`. |
|
||||
| 5.2 | ✅ **DONE (2026-07-30)** — AI usage metering | **P1** | **M** | 1.5 | No quota, no tracking, no ceiling. Hard blocker for Phase 7; a cost risk today with `AI_PROVIDER=gemini`. |
|
||||
| 5.3 | ✅ **DONE (2026-07-30)** — surfaced optional CV generation inside the add-job wizard | **P2** | **S** | 1.4 | The target workflow says "Generate CV if needed" at step 3. `POST /generate-tailored-cv-draft` exists but only post-save. |
|
||||
| 5.4 | ✅ **DONE** — keyword-gap analysis on match score | **P2** | **M** | none | `JobCvMatchService` + `/match-score` exist. Gap analysis is the specific thing people pay Jobscan $49.95/mo for. |
|
||||
| 5.5 | ✅ **DONE (2026-07-30)** — wrote ADR-004 (AI provider system) | **P2** | **S** | 5.1 | 0-byte file naming a real decision. |
|
||||
|
||||
@@ -31,10 +31,17 @@ beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
mockedApi.get.mockImplementation((url: string) => {
|
||||
if (url.includes("/modules")) return Promise.resolve({ data: { modules: [], provider: "gemini" } } as any);
|
||||
if (url === "/ai/usage") return Promise.resolve({ data: { currentMonth: { calls: 3, inputCharacters: 100, outputCharacters: 50, estimatedTokens: 38 }, allTime: { calls: 8, inputCharacters: 300, outputCharacters: 120, estimatedTokens: 105 } } } as any);
|
||||
return Promise.resolve({ data: [] } as any); // history
|
||||
});
|
||||
});
|
||||
|
||||
test("shows transparent monthly usage", async () => {
|
||||
renderPanel();
|
||||
expect(await screen.findByText(/approximately/)).toHaveTextContent("3 runs");
|
||||
expect(screen.getByText(/approximately/)).toHaveTextContent("38 tokens");
|
||||
});
|
||||
|
||||
test("renders modules and generates a suggestion into history", async () => {
|
||||
mockedApi.post.mockResolvedValueOnce({
|
||||
data: { id: 1, module: "job-analysis", mode: null, title: "Job analysis", provider: "gemini", result: { text: "**Company**\nAcme" }, createdAtUtc: new Date().toISOString() },
|
||||
|
||||
@@ -10,6 +10,11 @@ export type AiInteraction = {
|
||||
createdAtUtc: string;
|
||||
};
|
||||
|
||||
export type AiUsage = {
|
||||
currentMonth: { calls: number; inputCharacters: number; outputCharacters: number; estimatedTokens: number };
|
||||
allTime: { calls: number; inputCharacters: number; outputCharacters: number; estimatedTokens: number };
|
||||
};
|
||||
|
||||
export const AI_MODULES: { key: string; label: string; blurb: string }[] = [
|
||||
{ key: "job-analysis", label: "Job Analysis", blurb: "Break down the advert: skills, requirements, salary, work model, interview topics." },
|
||||
{ key: "career-match", label: "Career Match", blurb: "Your profile vs the advert: strengths, gaps, match %, suggested improvements." },
|
||||
@@ -21,6 +26,7 @@ export const AI_MODULES: { key: string; label: string; blurb: string }[] = [
|
||||
export const COVER_LETTER_MODES = ["professional", "friendly", "short", "detailed", "modern", "traditional"];
|
||||
|
||||
export const aiWorkspaceApi = {
|
||||
usage: () => api.get<AiUsage>("/ai/usage").then((r) => r.data),
|
||||
modules: (jobId: number) => api.get<{ modules: string[]; provider: string }>(`/jobapplications/${jobId}/ai/modules`).then((r) => r.data),
|
||||
generate: (jobId: number, body: { module: string; mode?: string; extraContext?: string }) =>
|
||||
api.post<AiInteraction>(`/jobapplications/${jobId}/ai/generate`, body).then((r) => r.data),
|
||||
|
||||
@@ -14,7 +14,7 @@ import ReplayIcon from "@mui/icons-material/Replay";
|
||||
import { getApiErrorMessage } from "../api";
|
||||
import { useToast } from "../toast";
|
||||
import Markdown from "./Markdown";
|
||||
import { AI_MODULES, AiInteraction, COVER_LETTER_MODES, aiWorkspaceApi } from "../aiWorkspace";
|
||||
import { AI_MODULES, AiInteraction, AiUsage, COVER_LETTER_MODES, aiWorkspaceApi } from "../aiWorkspace";
|
||||
|
||||
// Phase 5 — the central AI Workspace for one job application. Every result is a suggestion the user
|
||||
// reviews and copies; nothing is applied automatically.
|
||||
@@ -24,6 +24,7 @@ export default function AiWorkspacePanel({ jobId }: { jobId: number }) {
|
||||
const [mode, setMode] = useState("professional");
|
||||
const [extra, setExtra] = useState("");
|
||||
const [provider, setProvider] = useState("");
|
||||
const [usage, setUsage] = useState<AiUsage | null>(null);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const [current, setCurrent] = useState<AiInteraction | null>(null);
|
||||
const [history, setHistory] = useState<AiInteraction[]>([]);
|
||||
@@ -44,6 +45,7 @@ export default function AiWorkspacePanel({ jobId }: { jobId: number }) {
|
||||
|
||||
useEffect(() => {
|
||||
aiWorkspaceApi.modules(jobId).then((r) => setProvider(r.provider)).catch(() => undefined);
|
||||
aiWorkspaceApi.usage().then(setUsage).catch(() => undefined);
|
||||
loadHistory();
|
||||
}, [jobId, loadHistory]);
|
||||
|
||||
@@ -54,6 +56,7 @@ export default function AiWorkspacePanel({ jobId }: { jobId: number }) {
|
||||
const res = await aiWorkspaceApi.generate(jobId, { module, mode: module === "cover-letter" ? mode : undefined, extraContext: extra || undefined });
|
||||
setCurrent(res);
|
||||
setHistory((h) => [res, ...h]);
|
||||
aiWorkspaceApi.usage().then(setUsage).catch(() => undefined);
|
||||
} catch (err) {
|
||||
toast(getApiErrorMessage(err, "AI generation failed."), "error");
|
||||
} finally {
|
||||
@@ -85,6 +88,7 @@ export default function AiWorkspacePanel({ jobId }: { jobId: number }) {
|
||||
<Alert severity="info" sx={{ py: 0.5 }}>
|
||||
AI suggestions never change your profile, CVs, or this application. Review, then copy what you want to keep.
|
||||
{provider && <> Provider: <strong>{provider}</strong>.</>}
|
||||
{usage && <> This month: <strong>{usage.currentMonth.calls}</strong> runs · approximately <strong>{usage.currentMonth.estimatedTokens.toLocaleString()}</strong> tokens.</>}
|
||||
</Alert>
|
||||
|
||||
<Box sx={{ display: "flex", flexWrap: "wrap", gap: 1 }}>
|
||||
|
||||
Reference in New Issue
Block a user