fix(cv): gate AI proposals on grounding and source freshness

2026-10-08 14:28:06 +02:00
parent d8d262e4d1
commit 2398ff354e
27 changed files with 2553 additions and 157 deletions
@@ -0,0 +1,130 @@
+import { expect, test, type APIRequestContext } from "@playwright/test";
+
+const apiUrl = `${process.env.CV_AI_READINESS_API_URL}/api`;
+const providerUrl = process.env.CV_AI_READINESS_PROVIDER_URL!;
+const password = "E2ePassword123!";
+const source = "Built and maintained internal web applications.";
+const rewrite = "Developed and maintained internal web applications.";
+
+async function authenticate(api: APIRequestContext, email: string) {
+  expect((await api.post(`${apiUrl}/auth/login`, { data: { email, password } })).status()).toBe(200);
+  const me = await (await api.get(`${apiUrl}/auth/me`)).json();
+  expect(me.entitlements.ai).toBe(true);
+  const csrf = (await api.storageState()).cookies.find(cookie => cookie.name === "XSRF-TOKEN")!.value;
+  return { "X-CSRF-TOKEN": csrf, "X-Session-Context": me.sessionContext as string };
+}
+
+// The loopback stub supplies synthetic provider judgments. This proves transport, validation,
+// ownership and persistence contracts; it does not measure a live model's factual accuracy.
+test("bound CV proposals use real owned saves and reject fabricated or stale output without source/history mutation", async ({ request, playwright }, testInfo) => {
+  const headers = await authenticate(request, "cv-ai-readiness@example.test");
+  expect((await request.post(`${providerUrl}/fixture/reset`)).status()).toBe(200);
+  const profile = { version: "1", contact: { fullName: "Native AI Fixture", headline: source }, summary: [source] };
+  expect((await request.put(`${apiUrl}/career/profile`, { headers, data: { profile } })).status()).toBe(200);
+
+  const create = async (name: string) => {
+    const response = await request.post(`${apiUrl}/cv/variants`, { headers, data: { name, settings: { themeId: "minimal", language: "en", sections: [{ key: "summary" }] } } });
+    expect(response.status()).toBe(200);
+    return response.json();
+  };
+  const snapshot = async (id: number) => {
+    const responses = await Promise.all([
+      request.get(`${apiUrl}/cv/variants/${id}`), request.get(`${apiUrl}/cv/variants/${id}/versions`),
+      request.get(`${apiUrl}/career/profile`), request.get(`${apiUrl}/career/profile/versions`),
+    ]);
+    for (const response of responses) expect(response.status()).toBe(200);
+    const [variant, history, canonical, canonicalHistory] = await Promise.all(responses.map(response => response.json()));
+    return { variant, history, canonical, canonicalHistory };
+  };
+  const generate = async (cv: { id: number; version: number }, context: string, action = "improve") => request.post(`${apiUrl}/cv/ai/assist`, {
+    headers, data: { variantId: cv.id, expectedVersion: cv.version, action, language: "en", context,
+      ...(action.startsWith("document-") ? {} : { targetId: "section:summary", text: source }) },
+  });
+  const apply = (id: number, proposalToken: string) => request.post(`${apiUrl}/cv/variants/${id}/ai/apply`, { headers, data: { proposalToken } });
+
+  const owned = await create("Accepted section proposal");
+  const original = await snapshot(owned.id);
+  const generated = await generate(owned, "Fixture case: supported section.");
+  expect(generated.status()).toBe(200);
+  const proposal = await generated.json();
+  expect(proposal).toMatchObject({ original: source, result: rewrite, proposalToken: expect.any(String) });
+  expect(proposal.proposalToken.length).toBeGreaterThan(20);
+  expect(await snapshot(owned.id)).toEqual(original);
+  const accepted = await apply(owned.id, proposal.proposalToken);
+  expect(accepted.status()).toBe(200);
+  const saved = await accepted.json();
+  expect(saved.id).toBe(owned.id);
+  expect(saved.version).toBe(owned.version + 1);
+  expect(saved.settings.sections.find((section: { key: string }) => section.key === "summary").items).toEqual([rewrite]);
+  const persisted = await snapshot(owned.id);
+  expect(persisted.variant).toEqual(saved);
+  expect(persisted.history.map((entry: { version: number }) => entry.version)).toEqual([2, 1]);
+  expect(persisted.history[0].source).toBe("ai:improve");
+  expect(persisted.canonical).toEqual(original.canonical);
+  expect(persisted.canonicalHistory).toEqual(original.canonicalHistory);
+
+  const other = await playwright.request.newContext();
+  try {
+    expect((await request.post(`${apiUrl}/users`, { headers, data: { email: "cv-ai-other@example.test", password, roles: ["Premium"] } })).status()).toBe(200);
+    const otherHeaders = await authenticate(other, "cv-ai-other@example.test");
+    expect((await other.get(`${apiUrl}/cv/variants/${owned.id}`)).status()).toBe(404);
+    expect((await other.post(`${apiUrl}/cv/variants/${owned.id}/ai/apply`, { headers: otherHeaders, data: { proposalToken: proposal.proposalToken } })).status()).toBe(404);
+    expect(await snapshot(owned.id)).toEqual(persisted);
+  } finally { await other.dispose(); }
+
+  const documentCv = await create("Accepted document proposal");
+  const documentBefore = await snapshot(documentCv.id);
+  const documentGenerated = await generate(documentCv, "Fixture case: supported document.", "document-improve");
+  expect(documentGenerated.status()).toBe(200);
+  const documentProposal = await documentGenerated.json();
+  const originalBlocks = JSON.parse(documentProposal.original);
+  const proposedBlocks = JSON.parse(documentProposal.result);
+  expect(proposedBlocks.map((block: { id: string }) => block.id)).toEqual(originalBlocks.map((block: { id: string }) => block.id));
+  expect(await snapshot(documentCv.id)).toEqual(documentBefore);
+  expect((await apply(documentCv.id, documentProposal.proposalToken)).status()).toBe(200);
+  const documentSaved = await snapshot(documentCv.id);
+  expect(documentSaved.variant.version).toBe(2);
+  expect(documentSaved.variant.settings.headline).toBe(rewrite);
+  expect(documentSaved.canonical).toEqual(documentBefore.canonical);
+  expect(documentSaved.canonicalHistory).toEqual(documentBefore.canonicalHistory);
+
+  expect((await request.post(`${providerUrl}/fixture/mode`, { data: { mode: "fabricated" } })).status()).toBe(200);
+  const fabricatedCv = await create("Rejected fabricated proposal");
+  const fabricatedBefore = await snapshot(fabricatedCv.id);
+  const fabricated = await generate(fabricatedCv, "Fixture case: fabricated provider output.");
+  expect(fabricated.status()).toBe(422);
+  expect(await fabricated.json()).toEqual({ code: "cv_ai_grounding_unproven" });
+  expect(await snapshot(fabricatedCv.id)).toEqual(fabricatedBefore);
+  expect((await request.post(`${providerUrl}/fixture/mode`, { data: { mode: "supported" } })).status()).toBe(200);
+
+  const staleCv = await create("Rejected stale CV proposal");
+  const cvProposalResponse = await generate(staleCv, "Fixture case: stale CV.");
+  expect(cvProposalResponse.status()).toBe(200);
+  const cvProposal = await cvProposalResponse.json();
+  expect((await request.put(`${apiUrl}/cv/variants/${staleCv.id}`, { headers, data: {
+    expectedVersion: staleCv.version, settings: { ...staleCv.settings, headline: "Newer manual CV headline" }, source: "manual",
+  } })).status()).toBe(200);
+  const newerCv = await snapshot(staleCv.id);
+  const cvConflict = await apply(staleCv.id, cvProposal.proposalToken);
+  expect(cvConflict.status()).toBe(409);
+  expect(await cvConflict.json()).toEqual({ code: "cv_ai_source_changed" });
+  expect(await snapshot(staleCv.id)).toEqual(newerCv);
+
+  const staleProfileCv = await create("Rejected stale canonical proposal");
+  const profileProposalResponse = await generate(staleProfileCv, "Fixture case: stale canonical profile.");
+  expect(profileProposalResponse.status()).toBe(200);
+  const profileProposal = await profileProposalResponse.json();
+  expect((await request.put(`${apiUrl}/career/profile`, { headers, data: { profile: { ...profile, summary: ["Newer canonical user content."] } } })).status()).toBe(200);
+  const newerProfile = await snapshot(staleProfileCv.id);
+  expect(newerProfile.variant.version).toBe(1);
+  const profileConflict = await apply(staleProfileCv.id, profileProposal.proposalToken);
+  expect(profileConflict.status()).toBe(409);
+  expect(await profileConflict.json()).toEqual({ code: "cv_ai_source_changed" });
+  expect(await snapshot(staleProfileCv.id)).toEqual(newerProfile);
+
+  const state = await (await request.get(`${providerUrl}/fixture/state`)).json();
+  expect(state.counts.rewrite).toBe(5);
+  expect(state.counts.evaluate).toBe(5);
+  expect(state.calls.some((call: { path: string; body: { output_mode?: string } }) => call.path === "/cv/rewrite" && call.body.output_mode === "cv_blocks_json")).toBe(true);
+  await testInfo.attach("synthetic-provider-contract-calls", { body: JSON.stringify(state, null, 2), contentType: "application/json" });
+});