feat(admin): add safe processing monitor and operation correlation

2026-10-28 19:31:32 +01:00
parent 1f87301ee9
commit d52403dba8
19 changed files with 1064 additions and 6 deletions
@@ -0,0 +1,201 @@
+import React, { useEffect, useRef, useState } from 'react';
+import { Alert, Box, Button, Chip, Stack, Table, TableBody, TableCell, TableContainer, TableHead, TableRow, TextField, Typography } from '@mui/material';
+import { api } from '../api';
+import { AUTH_USER_CHANGED_EVENT, getAuthUserKey } from '../auth';
+import { isSessionInvalidated, SESSION_INVALIDATED_EVENT } from '../sessionBoundary';
+import { useI18n } from '../i18n/I18nProvider';
+import type { TranslationKey } from '../i18n/translations';
+import { PageHeader, PageSection } from '../layout/PageLayout';
+
+type Item = {
+  id: string; owner: { id: string; name: string | null; email: string | null }; taskType: string; status: string; stage: string | null;
+  attemptCount: number; maxAttempts: number; createdAtUtc: string; startedAtUtc: string | null; completedAtUtc: string | null;
+  lastHeartbeatAtUtc: string | null; durationSeconds: number; worker: string; workerEnabled: boolean; failureCode: string | null;
+  job: { id: number; title: string; company: string | null } | null;
+  document: { name: string; type: string | null; bytes: number; pages: number | null; state: string | null } | null;
+};
+type Listing = { items: Item[]; total: number; counts: Record<string, number>; workers: { name: string; flagEnabled: boolean; individualOperationsRecorded: boolean }[] };
+type Event = { id: string; atUtc: string; kind: string; status: string; stage: string | null; attempt: number; failureCode: string | null };
+type Events = { items: Event[]; hasMore: boolean; retentionDays: number };
+const statuses: Record<string, TranslationKey> = { queued: 'operationsStatusQueued', running: 'operationsStatusRunning', waiting_for_retry: 'operationsStatusWaitingForRetry', waiting_for_external_fallback: 'operationsStatusWaitingForFallback', succeeded: 'operationsStatusSucceeded', failed: 'operationsStatusFailed', cancelled: 'operationsStatusCancelled' };
+const tasks: Record<string, TranslationKey> = { 'cv.process': 'operationsTaskCv', 'strategy.snapshot': 'operationsTaskStrategy' };
+const stages: Record<string, TranslationKey> = { cv_validating_file: 'cvImportStageValidating', cv_extracting: 'cvImportStageExtracting', cv_identifying_sections: 'cvImportStageSections', cv_analysing: 'cvImportStageAnalysing', cv_validating_details: 'cvImportStageDetails', cv_preparing_review: 'cvImportStagePreparing', cv_review_ready: 'cvImportStageReady' };
+const kinds: Record<string, TranslationKey> = { Created: 'processingEventCreated', Claimed: 'processingEventClaimed', StageChanged: 'processingEventStageChanged', Completed: 'processingEventCompleted', RetryScheduled: 'processingEventRetryScheduled', Failed: 'processingEventFailed', CancellationRequested: 'processingEventCancellationRequested', Cancelled: 'processingEventCancelled', Retried: 'processingEventRetried', LeaseRecovered: 'processingEventLeaseRecovered', Expired: 'processingEventExpired' };
+const workers: Record<string, TranslationKey> = { AiOperations: 'processingWorkerAi', CvDocumentImports: 'processingWorkerDocument', JobEnrichment: 'processingWorkerEnrichment', Rules: 'processingWorkerRules', FollowUpReminders: 'processingWorkerReminders', DailyExport: 'processingWorkerExport', SavedJobSearchAlerts: 'processingWorkerSearch' };
+
+export default function AdminProcessingPage() {
+  const { t, language } = useI18n();
+  const [listing, setListing] = useState<Listing | null>(null);
+  const [selected, select] = useState<string | null>(null);
+  const [detail, setDetail] = useState<Item | null>(null);
+  const [events, setEvents] = useState<Event[]>([]);
+  const [hasOlder, setHasOlder] = useState(false);
+  const [before, setBefore] = useState<string | null>(null);
+  const [filters, setFilters] = useState({ status: '', taskType: '', owner: '' });
+  const [inputs, setInputs] = useState(filters);
+  const [page, setPage] = useState(1);
+  const [paused, setPaused] = useState(false);
+  const pausedRef = useRef(false);
+  const [refresh, setRefresh] = useState(0);
+  const [identity, setIdentity] = useState(getAuthUserKey);
+  const [busy, setBusy] = useState(true);
+  const [error, setError] = useState<'denied' | 'unavailable' | null>(null);
+  const [updated, setUpdated] = useState<string | null>(null);
+  const [copy, setCopy] = useState<'copied' | 'failed' | null>(null);
+  const detailHeading = useRef<HTMLHeadingElement>(null);
+  const focusDetail = useRef(false);
+
+  useEffect(() => {
+    const changed = () => {
+      setListing(null); setDetail(null); setEvents([]); select(null); setBefore(null);
+      setIdentity(getAuthUserKey()); setRefresh(value => value + 1);
+    };
+    window.addEventListener(AUTH_USER_CHANGED_EVENT, changed);
+    window.addEventListener(SESSION_INVALIDATED_EVENT, changed);
+    return () => { window.removeEventListener(AUTH_USER_CHANGED_EVENT, changed); window.removeEventListener(SESSION_INVALIDATED_EVENT, changed); };
+  }, []);
+
+  useEffect(() => {
+    if (isSessionInvalidated()) return;
+    let controller = new AbortController();
+    let stopped = false;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+    let cursor: string | undefined;
+    let eventWindow: Event[] = [];
+    const current = () => !stopped && identity === getAuthUserKey() && !isSessionInvalidated();
+    const load = async () => {
+      if (!current()) return;
+      const request = new AbortController();
+      controller = request;
+      const options = { signal: request.signal, timeout: 15000 };
+      setBusy(true);
+      let retry = true;
+      let delay = 5000;
+      try {
+        const query = new URLSearchParams({ page: String(page), pageSize: '25' });
+        for (const [key, value] of Object.entries(filters)) if (value) query.set(key, value);
+        const eventQuery = new URLSearchParams({ take: '100' });
+        if (before) eventQuery.set('before', before);
+        else if (cursor) eventQuery.set('after', cursor);
+        const [listResponse, detailResponse, eventResponse] = await Promise.all([
+          api.get<Listing>(`/admin/processing?${query}`, options),
+          selected ? api.get<Item>(`/admin/processing/${selected}`, options) : Promise.resolve(null),
+          selected ? api.get<Events>(`/admin/processing/${selected}/events?${eventQuery}`, options) : Promise.resolve(null),
+        ]);
+        if (!current() || request.signal.aborted) return;
+        const list = listResponse.data;
+        if (!list || !Array.isArray(list.items) || list.items.length > 100 || !Array.isArray(list.workers) || list.workers.length > 32
+          || !list.counts || typeof list.counts !== 'object' || !Number.isSafeInteger(list.total) || list.total < 0
+          || (detailResponse && (detailResponse.data?.id !== selected || typeof detailResponse.data?.owner?.id !== 'string'))
+          || (eventResponse && (!Array.isArray(eventResponse.data?.items) || eventResponse.data.items.length > 100))) throw new Error('Invalid processing response');
+        setListing(listResponse.data); setDetail(detailResponse?.data ?? null);
+        if (eventResponse) {
+          const next = eventResponse.data.items;
+          const append = Boolean(cursor) && !before;
+          const combined = append ? [...new Map([...eventWindow, ...next].map(item => [item.id, item])).values()] : next;
+          eventWindow = combined.slice(-200);
+          setEvents(eventWindow);
+          if (!append) setHasOlder(eventResponse.data.hasMore);
+          else if (combined.length > 200) setHasOlder(true);
+          if (next.length) cursor = next[next.length - 1].id;
+        } else setEvents([]);
+        setUpdated(new Date().toISOString()); setError(null);
+        if (focusDetail.current && detailResponse) {
+          focusDetail.current = false;
+          requestAnimationFrame(() => { if (current()) detailHeading.current?.focus(); });
+        }
+      } catch (cause) {
+        if (!current() || request.signal.aborted) return;
+        request.abort();
+        const status = (cause as { response?: { status?: number } }).response?.status;
+        if (status === 401 || status === 403) {
+          setListing(null); setDetail(null); setEvents([]); setError('denied'); retry = false;
+        } else { setError('unavailable'); delay = 15000; }
+      } finally {
+        if (current()) {
+          setBusy(false);
+          if (retry && !pausedRef.current && !before) timer = setTimeout(() => { if (!pausedRef.current) void load(); }, delay);
+        }
+      }
+    };
+    void load();
+    return () => { stopped = true; controller.abort(); clearTimeout(timer); };
+  }, [before, filters, identity, page, refresh, selected]);
+
+  const date = (value: string | null) => value ? new Date(value).toLocaleString(language === 'nb' ? 'nb-NO' : 'en-GB') : t('processingNotAvailable');
+  const state = (value: string) => t(statuses[value] ?? 'processingUnknown');
+  const stage = (value: string | null) => value && stages[value] ? t(stages[value]) : t('processingNotAvailable');
+  const failure = (value: string) => t(value === 'timeout' ? 'processingFailureTimeout' : value === 'lease_expired' ? 'processingFailureLease' : value === 'deadline_exceeded' ? 'processingFailureDeadline' : value === 'entitlement_changed' ? 'processingFailureAccess' : 'processingFailureGeneric');
+  const worker = (value: string) => t(value === 'document' ? 'processingWorkerDocument' : value === 'ai' ? 'processingWorkerAi' : 'processingWorkerUntracked');
+  const resetSelection = () => { select(null); setDetail(null); setEvents([]); setBefore(null); setCopy(null); };
+  const resume = () => { pausedRef.current = false; setPaused(false); setBefore(null); setRefresh(value => value + 1); };
+  const visibleListing = identity === getAuthUserKey() ? listing : null;
+  return <Box sx={{ display: 'grid', gap: 3, minWidth: 0, '& p, & td, & th': { overflowWrap: 'anywhere' }, '& button': { minHeight: 44 } }}>
+    <PageHeader level="h2" title={t('processingTitle')} description={t('processingSubtitle')} />
+    <Stack direction="row" useFlexGap flexWrap="wrap" spacing={1} alignItems="center">
+      <Button onClick={() => { if (paused) resume(); else { pausedRef.current = true; setPaused(true); } }}>{t(paused ? 'processingResume' : 'processingPause')}</Button>
+      <Button onClick={() => setRefresh(value => value + 1)} disabled={busy}>{t('refresh')}</Button>
+      <Typography role="status" aria-live="polite">{t(error === 'denied' ? 'processingDenied' : paused ? 'processingPaused' : error ? 'processingReconnecting' : 'processingConnected')}</Typography>
+      {updated ? <Typography variant="caption">{t('processingLastUpdated', { time: date(updated) })}</Typography> : null}
+    </Stack>
+    {error ? <Alert severity="error" action={error === 'unavailable' ? <Button onClick={() => setRefresh(value => value + 1)}>{t('retry')}</Button> : undefined}>{t(error === 'denied' ? 'processingDenied' : 'processingLoadFailed')}</Alert> : null}
+    {busy && !visibleListing ? <Typography role="status">{t('loading')}</Typography> : null}
+    <Box component="form" onSubmit={event => { event.preventDefault(); resetSelection(); setPage(1); setFilters({ ...inputs, owner: inputs.owner.trim() }); }}>
+      <Stack direction="row" useFlexGap flexWrap="wrap" spacing={2}>
+        <TextField select label={t('processingStatus')} value={inputs.status} onChange={event => setInputs(value => ({ ...value, status: event.target.value }))} slotProps={{ select: { native: true } }}>
+          <option value="">{t('processingAll')}</option>{Object.keys(statuses).map(value => <option key={value} value={value}>{state(value)}</option>)}
+        </TextField>
+        <TextField select label={t('processingTask')} value={inputs.taskType} onChange={event => setInputs(value => ({ ...value, taskType: event.target.value }))} slotProps={{ select: { native: true } }}>
+          <option value="">{t('processingAll')}</option>{Object.keys(tasks).map(value => <option key={value} value={value}>{t(tasks[value])}</option>)}<option value="other">{t('processingOther')}</option>
+        </TextField>
+        <TextField label={t('processingOwnerFilter')} value={inputs.owner} onChange={event => setInputs(value => ({ ...value, owner: event.target.value }))} slotProps={{ htmlInput: { maxLength: 255 } }} sx={{ flex: '1 1 14rem', minWidth: 0 }} />
+        <Button type="submit" variant="outlined">{t('processingApplyFilters')}</Button>
+      </Stack>
+    </Box>
+    {visibleListing ? <>
+      <Stack direction="row" useFlexGap flexWrap="wrap" spacing={1}>{Object.entries(visibleListing.counts).map(([key, count]) => <Chip key={key} label={`${state(key)}: ${count}`} />)}</Stack>
+      <PageSection title={t('processingJobs')} surface>
+        {!visibleListing.items.length ? <Typography>{t('processingNoJobs')}</Typography> : <TableContainer tabIndex={0} role="group" aria-label={t('processingJobs')} sx={{ overflowX: 'auto', maxWidth: '100%' }}>
+          <Table aria-label={t('processingJobs')} sx={{ minWidth: 720 }}><TableHead><TableRow>{['processingTask', 'processingOwner', 'processingStatus', 'processingWorker', 'processingCreated'].map(key => <TableCell key={key}>{t(key as TranslationKey)}</TableCell>)}</TableRow></TableHead>
+            <TableBody>{visibleListing.items.map(item => <TableRow key={item.id} selected={selected === item.id}>
+              <TableCell component="th" scope="row"><Button aria-label={t('processingViewDetails', { id: item.id })} onClick={() => { setDetail(null); setEvents([]); setBefore(null); setCopy(null); focusDetail.current = true; select(item.id); }}>{t(tasks[item.taskType] ?? 'processingOther')}</Button><Typography variant="caption" display="block">{item.job?.title ?? item.document?.name ?? item.id.slice(0, 8)}</Typography></TableCell>
+              <TableCell><Button onClick={() => { const next = { ...filters, owner: item.owner.id }; setInputs(next); setFilters(next); setPage(1); resetSelection(); }}>{item.owner.name ?? item.owner.email ?? item.owner.id}</Button><Typography variant="caption" display="block">{item.owner.email}</Typography></TableCell>
+              <TableCell>{state(item.status)}<Typography variant="caption" display="block">{stage(item.stage)}</Typography></TableCell>
+              <TableCell>{worker(item.worker)}<Typography variant="caption" display="block">{t(item.workerEnabled ? 'processingEnabled' : 'processingDisabled')}</Typography></TableCell>
+              <TableCell><time dateTime={item.createdAtUtc}>{date(item.createdAtUtc)}</time></TableCell>
+            </TableRow>)}</TableBody></Table>
+        </TableContainer>}
+        <Stack direction="row" spacing={1} useFlexGap flexWrap="wrap" alignItems="center" sx={{ mt: 2 }}>
+          <Button disabled={page <= 1 || busy} onClick={() => { resetSelection(); setPage(value => value - 1); }}>{t('processingPrevious')}</Button>
+          <Typography>{t('processingPage', { page, total: Math.max(1, Math.ceil(visibleListing.total / 25)) })}</Typography>
+          <Button disabled={page * 25 >= visibleListing.total || busy} onClick={() => { resetSelection(); setPage(value => value + 1); }}>{t('processingNext')}</Button>
+        </Stack>
+      </PageSection>
+      <Box component="details"><Box component="summary" sx={{ p: 1, cursor: 'pointer' }}>{t('processingWorkerFlags')}</Box>
+        <Typography variant="body2">{t('processingWorkerFlagsHelp')}</Typography><Typography variant="body2">{t('processingAiDescription')}</Typography>
+        <Stack direction="row" useFlexGap flexWrap="wrap" spacing={1} sx={{ mt: 1 }}>{visibleListing.workers.map(item => <Chip key={item.name} label={`${t(workers[item.name] ?? 'processingWorkerUntracked')}: ${t(item.flagEnabled ? 'processingEnabled' : 'processingDisabled')}`} />)}</Stack>
+      </Box>
+    </> : null}
+    {detail && visibleListing ? <PageSection surface title={t('processingDetails')} titleRef={detailHeading}>
+      <Stack spacing={2}>
+        <Stack direction="row" useFlexGap flexWrap="wrap" spacing={1} alignItems="center">
+          <TextField label={t('processingCorrelation')} value={detail.id} slotProps={{ input: { readOnly: true } }} sx={{ flex: '1 1 20rem', minWidth: 0 }} />
+          <Button onClick={async () => { try { await navigator.clipboard.writeText(detail.id); setCopy('copied'); } catch { setCopy('failed'); } }}>{t('processingCopy')}</Button>
+          {copy ? <Typography role="status">{t(copy === 'copied' ? 'processingCopied' : 'processingCopyFailed')}</Typography> : null}
+        </Stack>
+        <Typography>{detail.owner.name ?? detail.owner.id} · {detail.owner.email} · {state(detail.status)} · {stage(detail.stage)}</Typography>
+        <Typography>{t('processingAttempts', { count: detail.attemptCount, max: detail.maxAttempts })} · {t('processingDuration', { count: Math.round(detail.durationSeconds) })}</Typography>
+        <Box component="dl" sx={{ m: 0, display: 'grid', gridTemplateColumns: { xs: '1fr', sm: 'auto 1fr' }, gap: 1 }}>
+          {([['processingCreated', detail.createdAtUtc], ['processingStarted', detail.startedAtUtc], ['processingCompleted', detail.completedAtUtc], ['processingLastSeen', detail.lastHeartbeatAtUtc]] as const).map(([key, value]) => <React.Fragment key={key}><Typography component="dt">{t(key)}</Typography><Typography component="dd" sx={{ m: 0 }}>{date(value)}</Typography></React.Fragment>)}
+        </Box>
+        {detail.failureCode ? <Alert severity="warning">{failure(detail.failureCode)}</Alert> : null}
+        {detail.job ? <Typography>{detail.job.title} · {detail.job.company} · #{detail.job.id}</Typography> : null}
+        {detail.document ? <Box><Typography>{t('processingSource')}: {detail.document.name}</Typography><Typography variant="body2">{detail.document.type} · {t('processingBytes', { count: detail.document.bytes })}{detail.document.pages ? ` · ${t('processingPages', { count: detail.document.pages })}` : ''}</Typography><Typography variant="body2">{detail.document.state === 'pending_review' ? t('cvImportStageReady') : detail.document.state === 'applied' ? t('processingSourceApplied') : detail.document.state === 'expired' ? t('processingSourceExpired') : state(detail.document.state ?? 'unknown')}</Typography></Box> : null}
+        <Typography component="h3" variant="h6">{t('processingEvents')}</Typography>
+        <Typography variant="body2">{t('processingRetention')}</Typography>
+        {!events.length ? <Typography>{t('processingNoEvents')}</Typography> : <Box component="ol" sx={{ m: 0, pl: 3 }}>{events.map(item => <Box component="li" key={item.id} sx={{ mb: 2 }}><Typography>{t(kinds[item.kind] ?? 'processingUnknown')} · {state(item.status)}</Typography><Typography variant="body2"><time dateTime={item.atUtc}>{date(item.atUtc)}</time> · {stage(item.stage)} · {t('processingAttempts', { count: item.attempt, max: detail.maxAttempts })}</Typography>{item.failureCode ? <Typography variant="body2">{failure(item.failureCode)}</Typography> : null}</Box>)}</Box>}
+        <Stack direction="row" spacing={1}><Button disabled={!hasOlder || busy || !events.length} onClick={() => { pausedRef.current = true; setPaused(true); setBefore(events[0].id); }}>{t('processingOlder')}</Button><Button disabled={busy} onClick={resume}>{t('processingLatest')}</Button></Stack>
+      </Stack>
+    </PageSection> : null}
+  </Box>;
+}