jobtrackingapp/job-tracker-ui/ai-tests/cv-ai-readiness.spec.ts

import { expect, test, type APIRequestContext } from "@playwright/test";

const apiUrl = `${process.env.CV_AI_READINESS_API_URL}/api`;
const providerUrl = process.env.CV_AI_READINESS_PROVIDER_URL!;
const password = "E2ePassword123!";
const source = "Built and maintained internal web applications.";
const rewrite = "Developed and maintained internal web applications.";

async function authenticate(api: APIRequestContext, email: string) {
  expect((await api.post(`${apiUrl}/auth/login`, { data: { email, password } })).status()).toBe(200);
  const me = await (await api.get(`${apiUrl}/auth/me`)).json();
  expect(me.entitlements.ai).toBe(true);
  const csrf = (await api.storageState()).cookies.find(cookie => cookie.name === "XSRF-TOKEN")!.value;
  return { "X-CSRF-TOKEN": csrf, "X-Session-Context": me.sessionContext as string };
}

// The loopback stub supplies synthetic provider judgments. This proves transport, validation,
// ownership and persistence contracts; it does not measure a live model's factual accuracy.
test("bound CV proposals use real owned saves and reject fabricated or stale output without source/history mutation", async ({ request, playwright }, testInfo) => {
  const headers = await authenticate(request, "cv-ai-readiness@example.test");
  expect((await request.post(`${providerUrl}/fixture/reset`)).status()).toBe(200);
  const profile = { version: "1", contact: { fullName: "Native AI Fixture", headline: source }, summary: [source] };
  expect((await request.put(`${apiUrl}/career/profile`, { headers, data: { profile } })).status()).toBe(200);

  const create = async (name: string) => {
    const response = await request.post(`${apiUrl}/cv/variants`, { headers, data: { name, settings: { themeId: "minimal", language: "en", sections: [{ key: "summary" }] } } });
    expect(response.status()).toBe(200);
    return response.json();
  };
  const snapshot = async (id: number) => {
    const responses = await Promise.all([
      request.get(`${apiUrl}/cv/variants/${id}`), request.get(`${apiUrl}/cv/variants/${id}/versions`),
      request.get(`${apiUrl}/career/profile`), request.get(`${apiUrl}/career/profile/versions`),
    ]);
    for (const response of responses) expect(response.status()).toBe(200);
    const [variant, history, canonical, canonicalHistory] = await Promise.all(responses.map(response => response.json()));
    return { variant, history, canonical, canonicalHistory };
  };
  const generate = async (cv: { id: number; version: number }, context: string, action = "improve") => request.post(`${apiUrl}/cv/ai/assist`, {
    headers, data: { variantId: cv.id, expectedVersion: cv.version, action, language: "en", context,
      ...(action.startsWith("document-") ? {} : { targetId: "section:summary", text: source }) },
  });
  const apply = (id: number, proposalToken: string) => request.post(`${apiUrl}/cv/variants/${id}/ai/apply`, { headers, data: { proposalToken } });

  const owned = await create("Accepted section proposal");
  const original = await snapshot(owned.id);
  const generated = await generate(owned, "Fixture case: supported section.");
  expect(generated.status()).toBe(200);
  const proposal = await generated.json();
  expect(proposal).toMatchObject({ original: source, result: rewrite, proposalToken: expect.any(String) });
  expect(proposal.proposalToken.length).toBeGreaterThan(20);
  expect(await snapshot(owned.id)).toEqual(original);
  const accepted = await apply(owned.id, proposal.proposalToken);
  expect(accepted.status()).toBe(200);
  const saved = await accepted.json();
  expect(saved.id).toBe(owned.id);
  expect(saved.version).toBe(owned.version + 1);
  expect(saved.settings.sections.find((section: { key: string }) => section.key === "summary").items).toEqual([rewrite]);
  const persisted = await snapshot(owned.id);
  expect(persisted.variant).toEqual(saved);
  expect(persisted.history.map((entry: { version: number }) => entry.version)).toEqual([2, 1]);
  expect(persisted.history[0].source).toBe("ai:improve");
  expect(persisted.canonical).toEqual(original.canonical);
  expect(persisted.canonicalHistory).toEqual(original.canonicalHistory);

  const other = await playwright.request.newContext();
  try {
    expect((await request.post(`${apiUrl}/users`, { headers, data: { email: "cv-ai-other@example.test", password, roles: ["Premium"] } })).status()).toBe(200);
    const otherHeaders = await authenticate(other, "cv-ai-other@example.test");
    expect((await other.get(`${apiUrl}/cv/variants/${owned.id}`)).status()).toBe(404);
    expect((await other.post(`${apiUrl}/cv/variants/${owned.id}/ai/apply`, { headers: otherHeaders, data: { proposalToken: proposal.proposalToken } })).status()).toBe(404);
    expect(await snapshot(owned.id)).toEqual(persisted);
  } finally { await other.dispose(); }

  const documentCv = await create("Accepted document proposal");
  const documentBefore = await snapshot(documentCv.id);
  const documentGenerated = await generate(documentCv, "Fixture case: supported document.", "document-improve");
  expect(documentGenerated.status()).toBe(200);
  const documentProposal = await documentGenerated.json();
  const originalBlocks = JSON.parse(documentProposal.original);
  const proposedBlocks = JSON.parse(documentProposal.result);
  expect(proposedBlocks.map((block: { id: string }) => block.id)).toEqual(originalBlocks.map((block: { id: string }) => block.id));
  expect(await snapshot(documentCv.id)).toEqual(documentBefore);
  expect((await apply(documentCv.id, documentProposal.proposalToken)).status()).toBe(200);
  const documentSaved = await snapshot(documentCv.id);
  expect(documentSaved.variant.version).toBe(2);
  expect(documentSaved.variant.settings.headline).toBe(rewrite);
  expect(documentSaved.canonical).toEqual(documentBefore.canonical);
  expect(documentSaved.canonicalHistory).toEqual(documentBefore.canonicalHistory);

  expect((await request.post(`${providerUrl}/fixture/mode`, { data: { mode: "fabricated" } })).status()).toBe(200);
  const fabricatedCv = await create("Rejected fabricated proposal");
  const fabricatedBefore = await snapshot(fabricatedCv.id);
  const fabricated = await generate(fabricatedCv, "Fixture case: fabricated provider output.");
  expect(fabricated.status()).toBe(422);
  expect(await fabricated.json()).toEqual({ code: "cv_ai_grounding_unproven" });
  expect(await snapshot(fabricatedCv.id)).toEqual(fabricatedBefore);
  expect((await request.post(`${providerUrl}/fixture/mode`, { data: { mode: "supported" } })).status()).toBe(200);

  const staleCv = await create("Rejected stale CV proposal");
  const cvProposalResponse = await generate(staleCv, "Fixture case: stale CV.");
  expect(cvProposalResponse.status()).toBe(200);
  const cvProposal = await cvProposalResponse.json();
  expect((await request.put(`${apiUrl}/cv/variants/${staleCv.id}`, { headers, data: {
    expectedVersion: staleCv.version, settings: { ...staleCv.settings, headline: "Newer manual CV headline" }, source: "manual",
  } })).status()).toBe(200);
  const newerCv = await snapshot(staleCv.id);
  const cvConflict = await apply(staleCv.id, cvProposal.proposalToken);
  expect(cvConflict.status()).toBe(409);
  expect(await cvConflict.json()).toEqual({ code: "cv_ai_source_changed" });
  expect(await snapshot(staleCv.id)).toEqual(newerCv);

  const staleProfileCv = await create("Rejected stale canonical proposal");
  const profileProposalResponse = await generate(staleProfileCv, "Fixture case: stale canonical profile.");
  expect(profileProposalResponse.status()).toBe(200);
  const profileProposal = await profileProposalResponse.json();
  expect((await request.put(`${apiUrl}/career/profile`, { headers, data: { profile: { ...profile, summary: ["Newer canonical user content."] } } })).status()).toBe(200);
  const newerProfile = await snapshot(staleProfileCv.id);
  expect(newerProfile.variant.version).toBe(1);
  const profileConflict = await apply(staleProfileCv.id, profileProposal.proposalToken);
  expect(profileConflict.status()).toBe(409);
  expect(await profileConflict.json()).toEqual({ code: "cv_ai_source_changed" });
  expect(await snapshot(staleProfileCv.id)).toEqual(newerProfile);

  const state = await (await request.get(`${providerUrl}/fixture/state`)).json();
  expect(state.counts.rewrite).toBe(5);
  expect(state.counts.evaluate).toBe(5);
  expect(state.calls.some((call: { path: string; body: { output_mode?: string } }) => call.path === "/cv/rewrite" && call.body.output_mode === "cv_blocks_json")).toBe(true);
  await testInfo.attach("synthetic-provider-contract-calls", { body: JSON.stringify(state, null, 2), contentType: "application/json" });
});

test("Production withholds CV review before provider calls or content writes while existing AI remains available", async ({ request, playwright }) => {
  const productionUrl = `${process.env.CV_AI_READINESS_PRODUCTION_API_URL}/api`;
  const api = await playwright.request.newContext();
  try {
    const login = await api.post(`${productionUrl}/auth/login`, { data: { email: "cv-ai-readiness@example.test", password } });
    expect(login.status()).toBe(200);
    // This is an isolated loopback API behind an HTTPS canonical origin. Keep its Secure cookies
    // unchanged and explicitly transport them across the test-only internal HTTP connection.
    const cookies = login.headersArray().filter(header => header.name.toLowerCase() === "set-cookie").map(header => header.value.split(";")[0]);
    const headers: Record<string, string> = { Cookie: cookies.join("; "), "X-CSRF-TOKEN": cookies.find(cookie => cookie.startsWith("XSRF-TOKEN="))!.slice("XSRF-TOKEN=".length) };
    const me = await api.get(`${productionUrl}/auth/me`, { headers });
    expect(me.status()).toBe(200);
    const account = await me.json();
    expect(account.entitlements.ai).toBe(true);
    headers["X-Session-Context"] = account.sessionContext;
    expect((await request.post(`${providerUrl}/fixture/reset`)).status()).toBe(200);

    const profile = { version: "1", contact: { fullName: "Production Gate Fixture", headline: source }, summary: [source] };
    expect((await api.put(`${productionUrl}/career/profile`, { headers, data: { profile } })).status()).toBe(200);
    const created = await api.post(`${productionUrl}/cv/variants`, { headers, data: { name: "Withheld AI review", settings: { themeId: "minimal", language: "en", sections: [{ key: "summary" }] } } });
    expect(created.status()).toBe(200);
    const cv = await created.json();
    const snapshot = async () => {
      const responses = await Promise.all([
        api.get(`${productionUrl}/cv/variants/${cv.id}`, { headers }), api.get(`${productionUrl}/cv/variants/${cv.id}/versions`, { headers }),
        api.get(`${productionUrl}/career/profile`, { headers }), api.get(`${productionUrl}/career/profile/versions`, { headers }),
      ]);
      for (const response of responses) expect(response.status()).toBe(200);
      return Promise.all(responses.map(response => response.json()));
    };
    const before = await snapshot();
    for (const action of ["improve", "document-improve"]) {
      const response = await api.post(`${productionUrl}/cv/ai/assist`, { headers, data: {
        variantId: cv.id, expectedVersion: cv.version, action, language: "en",
        ...(action.startsWith("document-") ? {} : { targetId: "section:summary", text: source }),
      } });
      expect(response.status()).toBe(503);
      expect(await response.json()).toEqual({ code: "cv_ai_unavailable" });
    }
    const applied = await api.post(`${productionUrl}/cv/variants/${cv.id}/ai/apply`, { headers, data: { proposalToken: "unused-proposal", result: rewrite } });
    expect(applied.status()).toBe(503);
    expect(await applied.json()).toEqual({ code: "cv_ai_unavailable" });
    expect(await snapshot()).toEqual(before);
    expect(await (await request.get(`${providerUrl}/fixture/state`)).json()).toEqual({ mode: "supported", counts: { rewrite: 0, evaluate: 0 }, calls: [] });

    expect((await api.get(`${productionUrl}/jobapplications/1/ai/modules`, { headers })).status()).toBe(200);
    const legacy = await api.post(`${productionUrl}/profile-cv/rewrite-section`, { headers, data: { sourceText: source, language: "en" } });
    expect(legacy.status()).toBe(200);
    expect((await legacy.json()).text).toBe(rewrite);
    expect(await snapshot()).toEqual(before);
    const state = await (await request.get(`${providerUrl}/fixture/state`)).json();
    expect(state.counts).toEqual({ rewrite: 1, evaluate: 0 });
  } finally { await api.dispose(); }
});