fix(admin): complete processing diagnostics and failure evidence
This commit is contained in:
cesnimda
@@ -1,5 +1,6 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import { Alert, Box, Button, Chip, Stack, Table, TableBody, TableCell, TableContainer, TableHead, TableRow, TextField, Typography } from '@mui/material';
|
||||
import { Alert, Box, Button, Chip, Link, Stack, Table, TableBody, TableCell, TableContainer, TableHead, TableRow, TextField, Typography } from '@mui/material';
|
||||
import { Link as RouterLink } from 'react-router-dom';
|
||||
import { api } from '../api';
|
||||
import { AUTH_USER_CHANGED_EVENT, getAuthUserKey } from '../auth';
|
||||
import { isSessionInvalidated, SESSION_INVALIDATED_EVENT } from '../sessionBoundary';
|
||||
@@ -11,8 +12,9 @@ type Item = {
|
||||
id: string; owner: { id: string; name: string | null; email: string | null }; taskType: string; status: string; stage: string | null;
|
||||
attemptCount: number; maxAttempts: number; createdAtUtc: string; startedAtUtc: string | null; completedAtUtc: string | null;
|
||||
lastHeartbeatAtUtc: string | null; durationSeconds: number; worker: string; workerEnabled: boolean; failureCode: string | null;
|
||||
job: { id: number; title: string; company: string | null } | null;
|
||||
document: { name: string; type: string | null; bytes: number; pages: number | null; state: string | null } | null;
|
||||
lastFailureAtUtc: string | null; lastFailureStage: string | null;
|
||||
job: { id: number; title: string; company: string | null; canOpen: boolean } | null;
|
||||
document: { importId: number; name: string; type: string | null; bytes: number; pages: number | null; state: string | null } | null;
|
||||
};
|
||||
type Listing = { items: Item[]; total: number; counts: Record<string, number>; workers: { name: string; flagEnabled: boolean; individualOperationsRecorded: boolean }[] };
|
||||
type Event = { id: string; atUtc: string; kind: string; status: string; stage: string | null; attempt: number; failureCode: string | null };
|
||||
@@ -22,6 +24,7 @@ const tasks: Record<string, TranslationKey> = { 'cv.process': 'operationsTaskCv'
|
||||
const stages: Record<string, TranslationKey> = { cv_validating_file: 'cvImportStageValidating', cv_extracting: 'cvImportStageExtracting', cv_identifying_sections: 'cvImportStageSections', cv_analysing: 'cvImportStageAnalysing', cv_validating_details: 'cvImportStageDetails', cv_preparing_review: 'cvImportStagePreparing', cv_review_ready: 'cvImportStageReady' };
|
||||
const kinds: Record<string, TranslationKey> = { Created: 'processingEventCreated', Claimed: 'processingEventClaimed', StageChanged: 'processingEventStageChanged', Completed: 'processingEventCompleted', RetryScheduled: 'processingEventRetryScheduled', Failed: 'processingEventFailed', CancellationRequested: 'processingEventCancellationRequested', Cancelled: 'processingEventCancelled', Retried: 'processingEventRetried', LeaseRecovered: 'processingEventLeaseRecovered', Expired: 'processingEventExpired' };
|
||||
const workers: Record<string, TranslationKey> = { AiOperations: 'processingWorkerAi', CvDocumentImports: 'processingWorkerDocument', JobEnrichment: 'processingWorkerEnrichment', Rules: 'processingWorkerRules', FollowUpReminders: 'processingWorkerReminders', DailyExport: 'processingWorkerExport', SavedJobSearchAlerts: 'processingWorkerSearch' };
|
||||
const failureCodes = ['timeout', 'lease_expired', 'deadline_exceeded', 'entitlement_changed', 'document_unavailable', 'sidecar_unavailable', 'upstream_unavailable', 'invalid_response', 'circuit_open', 'rate_limited', 'permission_denied', 'cancelled', 'upload_rejected', 'parser_failed', 'malformed_output', 'provider_unavailable', 'processing_failed'];
|
||||
|
||||
export default function AdminProcessingPage() {
|
||||
const { t, language } = useI18n();
|
||||
@@ -31,7 +34,7 @@ export default function AdminProcessingPage() {
|
||||
const [events, setEvents] = useState<Event[]>([]);
|
||||
const [hasOlder, setHasOlder] = useState(false);
|
||||
const [before, setBefore] = useState<string | null>(null);
|
||||
const [filters, setFilters] = useState({ status: '', taskType: '', owner: '' });
|
||||
const [filters, setFilters] = useState({ status: '', taskType: '', owner: '', q: '', jobId: '', worker: '', failureCode: '', createdFrom: '', createdBefore: '' });
|
||||
const [inputs, setInputs] = useState(filters);
|
||||
const [page, setPage] = useState(1);
|
||||
const [paused, setPaused] = useState(false);
|
||||
@@ -39,7 +42,7 @@ export default function AdminProcessingPage() {
|
||||
const [refresh, setRefresh] = useState(0);
|
||||
const [identity, setIdentity] = useState(getAuthUserKey);
|
||||
const [busy, setBusy] = useState(true);
|
||||
const [error, setError] = useState<'denied' | 'unavailable' | null>(null);
|
||||
const [error, setError] = useState<'denied' | 'unavailable' | 'invalid' | null>(null);
|
||||
const [updated, setUpdated] = useState<string | null>(null);
|
||||
const [copy, setCopy] = useState<'copied' | 'failed' | null>(null);
|
||||
const detailHeading = useRef<HTMLHeadingElement>(null);
|
||||
@@ -73,7 +76,7 @@ export default function AdminProcessingPage() {
|
||||
let delay = 5000;
|
||||
try {
|
||||
const query = new URLSearchParams({ page: String(page), pageSize: '25' });
|
||||
for (const [key, value] of Object.entries(filters)) if (value) query.set(key, value);
|
||||
for (const [key, value] of Object.entries(filters)) if (value) query.set(key, key === 'createdFrom' || key === 'createdBefore' ? `${value}T00:00:00Z` : value);
|
||||
const eventQuery = new URLSearchParams({ take: '100' });
|
||||
if (before) eventQuery.set('before', before);
|
||||
else if (cursor) eventQuery.set('after', cursor);
|
||||
@@ -110,7 +113,8 @@ export default function AdminProcessingPage() {
|
||||
const status = (cause as { response?: { status?: number } }).response?.status;
|
||||
if (status === 401 || status === 403) {
|
||||
setListing(null); setDetail(null); setEvents([]); setError('denied'); retry = false;
|
||||
} else { setError('unavailable'); delay = 15000; }
|
||||
} else if (status === 400) { setError('invalid'); retry = false; }
|
||||
else { setError('unavailable'); delay = 15000; }
|
||||
} finally {
|
||||
if (current()) {
|
||||
setBusy(false);
|
||||
@@ -135,10 +139,10 @@ export default function AdminProcessingPage() {
|
||||
<Stack direction="row" useFlexGap flexWrap="wrap" spacing={1} alignItems="center">
|
||||
<Button onClick={() => { if (paused) resume(); else { pausedRef.current = true; setPaused(true); } }}>{t(paused ? 'processingResume' : 'processingPause')}</Button>
|
||||
<Button onClick={() => setRefresh(value => value + 1)} disabled={busy}>{t('refresh')}</Button>
|
||||
<Typography role="status" aria-live="polite">{t(error === 'denied' ? 'processingDenied' : paused ? 'processingPaused' : error ? 'processingReconnecting' : 'processingConnected')}</Typography>
|
||||
<Typography role="status" aria-live="polite">{t(error === 'denied' ? 'processingDenied' : error === 'invalid' ? 'processingInvalidFilters' : paused ? 'processingPaused' : error ? 'processingReconnecting' : 'processingConnected')}</Typography>
|
||||
{updated ? <Typography variant="caption">{t('processingLastUpdated', { time: date(updated) })}</Typography> : null}
|
||||
</Stack>
|
||||
{error ? <Alert severity="error" action={error === 'unavailable' ? <Button onClick={() => setRefresh(value => value + 1)}>{t('retry')}</Button> : undefined}>{t(error === 'denied' ? 'processingDenied' : 'processingLoadFailed')}</Alert> : null}
|
||||
{error ? <Alert severity="error" action={error === 'unavailable' ? <Button onClick={() => setRefresh(value => value + 1)}>{t('retry')}</Button> : undefined}>{t(error === 'denied' ? 'processingDenied' : error === 'invalid' ? 'processingInvalidFilters' : 'processingLoadFailed')}</Alert> : null}
|
||||
{busy && !visibleListing ? <Typography role="status">{t('loading')}</Typography> : null}
|
||||
<Box component="form" onSubmit={event => { event.preventDefault(); resetSelection(); setPage(1); setFilters({ ...inputs, owner: inputs.owner.trim() }); }}>
|
||||
<Stack direction="row" useFlexGap flexWrap="wrap" spacing={2}>
|
||||
@@ -149,6 +153,16 @@ export default function AdminProcessingPage() {
|
||||
<option value="">{t('processingAll')}</option>{Object.keys(tasks).map(value => <option key={value} value={value}>{t(tasks[value])}</option>)}<option value="other">{t('processingOther')}</option>
|
||||
</TextField>
|
||||
<TextField label={t('processingOwnerFilter')} value={inputs.owner} onChange={event => setInputs(value => ({ ...value, owner: event.target.value }))} slotProps={{ htmlInput: { maxLength: 255 } }} sx={{ flex: '1 1 14rem', minWidth: 0 }} />
|
||||
<TextField label={t('processingSearch')} value={inputs.q} onChange={event => setInputs(value => ({ ...value, q: event.target.value }))} slotProps={{ htmlInput: { maxLength: 200 } }} />
|
||||
<TextField type="number" label={t('processingApplicationId')} value={inputs.jobId} onChange={event => setInputs(value => ({ ...value, jobId: event.target.value }))} slotProps={{ htmlInput: { min: 1, max: 2147483647 } }} />
|
||||
<TextField type="date" label={t('processingCreatedFrom')} value={inputs.createdFrom} onChange={event => setInputs(value => ({ ...value, createdFrom: event.target.value }))} slotProps={{ inputLabel: { shrink: true } }} />
|
||||
<TextField type="date" label={t('processingCreatedBefore')} value={inputs.createdBefore} onChange={event => setInputs(value => ({ ...value, createdBefore: event.target.value }))} slotProps={{ inputLabel: { shrink: true } }} />
|
||||
<TextField select label={t('processingWorkerFilter')} value={inputs.worker} onChange={event => setInputs(value => ({ ...value, worker: event.target.value }))} slotProps={{ select: { native: true } }}>
|
||||
<option value="">{t('processingAll')}</option>{['document', 'ai', 'untracked'].map(value => <option key={value} value={value}>{worker(value)}</option>)}
|
||||
</TextField>
|
||||
<TextField select label={t('processingFailureCode')} value={inputs.failureCode} onChange={event => setInputs(value => ({ ...value, failureCode: event.target.value }))} slotProps={{ select: { native: true } }}>
|
||||
<option value="">{t('processingAll')}</option>{failureCodes.map(value => <option key={value} value={value}>{value}</option>)}
|
||||
</TextField>
|
||||
<Button type="submit" variant="outlined">{t('processingApplyFilters')}</Button>
|
||||
</Stack>
|
||||
</Box>
|
||||
@@ -156,19 +170,20 @@ export default function AdminProcessingPage() {
|
||||
<Stack direction="row" useFlexGap flexWrap="wrap" spacing={1}>{Object.entries(visibleListing.counts).map(([key, count]) => <Chip key={key} label={`${state(key)}: ${count}`} />)}</Stack>
|
||||
<PageSection title={t('processingJobs')} surface>
|
||||
{!visibleListing.items.length ? <Typography>{t('processingNoJobs')}</Typography> : <TableContainer tabIndex={0} role="group" aria-label={t('processingJobs')} sx={{ overflowX: 'auto', maxWidth: '100%' }}>
|
||||
<Table aria-label={t('processingJobs')} sx={{ minWidth: 720 }}><TableHead><TableRow>{['processingTask', 'processingOwner', 'processingStatus', 'processingWorker', 'processingCreated'].map(key => <TableCell key={key}>{t(key as TranslationKey)}</TableCell>)}</TableRow></TableHead>
|
||||
<Table aria-label={t('processingJobs')} sx={{ minWidth: 800 }}><TableHead><TableRow>{['processingTask', 'processingOwner', 'processingStatus', 'processingWorker', 'processingCreated', 'processingElapsed'].map(key => <TableCell key={key}>{t(key as TranslationKey)}</TableCell>)}</TableRow></TableHead>
|
||||
<TableBody>{visibleListing.items.map(item => <TableRow key={item.id} selected={selected === item.id}>
|
||||
<TableCell component="th" scope="row"><Button aria-label={t('processingViewDetails', { id: item.id })} onClick={() => { setDetail(null); setEvents([]); setBefore(null); setCopy(null); focusDetail.current = true; select(item.id); }}>{t(tasks[item.taskType] ?? 'processingOther')}</Button><Typography variant="caption" display="block">{item.job?.title ?? item.document?.name ?? item.id.slice(0, 8)}</Typography></TableCell>
|
||||
<TableCell><Button onClick={() => { const next = { ...filters, owner: item.owner.id }; setInputs(next); setFilters(next); setPage(1); resetSelection(); }}>{item.owner.name ?? item.owner.email ?? item.owner.id}</Button><Typography variant="caption" display="block">{item.owner.email}</Typography></TableCell>
|
||||
<TableCell>{state(item.status)}<Typography variant="caption" display="block">{stage(item.stage)}</Typography></TableCell>
|
||||
<TableCell>{state(item.status)}<Typography variant="caption" display="block">{stage(item.stage)}</Typography>{item.failureCode && ['failed', 'waiting_for_retry', 'waiting_for_external_fallback'].includes(item.status) ? <Chip size="small" color="warning" label={failure(item.failureCode)} /> : null}</TableCell>
|
||||
<TableCell>{worker(item.worker)}<Typography variant="caption" display="block">{t(item.workerEnabled ? 'processingEnabled' : 'processingDisabled')}</Typography></TableCell>
|
||||
<TableCell><time dateTime={item.createdAtUtc}>{date(item.createdAtUtc)}</time></TableCell>
|
||||
<TableCell>{t('processingDuration', { count: Math.round(item.durationSeconds) })}</TableCell>
|
||||
</TableRow>)}</TableBody></Table>
|
||||
</TableContainer>}
|
||||
<Stack direction="row" spacing={1} useFlexGap flexWrap="wrap" alignItems="center" sx={{ mt: 2 }}>
|
||||
<Button disabled={page <= 1 || busy} onClick={() => { resetSelection(); setPage(value => value - 1); }}>{t('processingPrevious')}</Button>
|
||||
<Typography>{t('processingPage', { page, total: Math.max(1, Math.ceil(visibleListing.total / 25)) })}</Typography>
|
||||
<Button disabled={page * 25 >= visibleListing.total || busy} onClick={() => { resetSelection(); setPage(value => value + 1); }}>{t('processingNext')}</Button>
|
||||
<Typography>{t('processingPage', { page, total: Math.max(1, Math.min(1000, Math.ceil(visibleListing.total / 25))) })}</Typography>
|
||||
<Button disabled={page >= 1000 || page * 25 >= visibleListing.total || busy} onClick={() => { resetSelection(); setPage(value => value + 1); }}>{t('processingNext')}</Button>
|
||||
</Stack>
|
||||
</PageSection>
|
||||
<Box component="details"><Box component="summary" sx={{ p: 1, cursor: 'pointer' }}>{t('processingWorkerFlags')}</Box>
|
||||
@@ -184,13 +199,14 @@ export default function AdminProcessingPage() {
|
||||
{copy ? <Typography role="status">{t(copy === 'copied' ? 'processingCopied' : 'processingCopyFailed')}</Typography> : null}
|
||||
</Stack>
|
||||
<Typography>{detail.owner.name ?? detail.owner.id} · {detail.owner.email} · {state(detail.status)} · {stage(detail.stage)}</Typography>
|
||||
<TextField label={t('processingSubmittingUserId')} value={detail.owner.id} slotProps={{ input: { readOnly: true } }} />
|
||||
<Typography>{t('processingAttempts', { count: detail.attemptCount, max: detail.maxAttempts })} · {t('processingDuration', { count: Math.round(detail.durationSeconds) })}</Typography>
|
||||
<Box component="dl" sx={{ m: 0, display: 'grid', gridTemplateColumns: { xs: '1fr', sm: 'auto 1fr' }, gap: 1 }}>
|
||||
{([['processingCreated', detail.createdAtUtc], ['processingStarted', detail.startedAtUtc], ['processingCompleted', detail.completedAtUtc], ['processingLastSeen', detail.lastHeartbeatAtUtc]] as const).map(([key, value]) => <React.Fragment key={key}><Typography component="dt">{t(key)}</Typography><Typography component="dd" sx={{ m: 0 }}>{date(value)}</Typography></React.Fragment>)}
|
||||
</Box>
|
||||
{detail.failureCode ? <Alert severity="warning">{failure(detail.failureCode)}</Alert> : null}
|
||||
{detail.job ? <Typography>{detail.job.title} · {detail.job.company} · #{detail.job.id}</Typography> : null}
|
||||
{detail.document ? <Box><Typography>{t('processingSource')}: {detail.document.name}</Typography><Typography variant="body2">{detail.document.type} · {t('processingBytes', { count: detail.document.bytes })}{detail.document.pages ? ` · ${t('processingPages', { count: detail.document.pages })}` : ''}</Typography><Typography variant="body2">{detail.document.state === 'pending_review' ? t('cvImportStageReady') : detail.document.state === 'applied' ? t('processingSourceApplied') : detail.document.state === 'expired' ? t('processingSourceExpired') : state(detail.document.state ?? 'unknown')}</Typography></Box> : null}
|
||||
{detail.failureCode ? <Alert severity="warning"><Typography>{!['failed', 'waiting_for_retry', 'waiting_for_external_fallback'].includes(detail.status) ? `${t('processingPreviousFailure')}: ` : ''}{failure(detail.failureCode)}</Typography><Typography variant="body2">{t('processingFailureCode')}: {detail.failureCode} · {t('processingFailureAt')}: {date(detail.lastFailureAtUtc)} · {stage(detail.lastFailureStage)}</Typography><Typography variant="body2">{t(detail.status === 'waiting_for_retry' ? 'processingRetryScheduled' : 'processingRetryUnknown')}</Typography></Alert> : null}
|
||||
{detail.job ? <Typography>{detail.job.canOpen ? <Link component={RouterLink} to={`/jobs/${detail.job.id}`}>{detail.job.title}</Link> : detail.job.title} · {detail.job.company} · #{detail.job.id}</Typography> : <Typography>{t('processingNoLinkedJob')}</Typography>}
|
||||
{detail.document ? <Box><Typography>{t('processingImportId')}: {detail.document.importId}</Typography><Typography>{t('processingSource')}: {detail.document.name}</Typography><Typography variant="body2">{detail.document.type} · {t('processingBytes', { count: detail.document.bytes })}{detail.document.pages ? ` · ${t('processingPages', { count: detail.document.pages })}` : ''}</Typography><Typography variant="body2">{detail.document.state === 'pending_review' ? t('cvImportStageReady') : detail.document.state === 'applied' ? t('processingSourceApplied') : detail.document.state === 'expired' ? t('processingSourceExpired') : state(detail.document.state ?? 'unknown')}</Typography></Box> : null}
|
||||
<Typography component="h3" variant="h6">{t('processingEvents')}</Typography>
|
||||
<Typography variant="body2">{t('processingRetention')}</Typography>
|
||||
{!events.length ? <Typography>{t('processingNoEvents')}</Typography> : <Box component="ol" sx={{ m: 0, pl: 3 }}>{events.map(item => <Box component="li" key={item.id} sx={{ mb: 2 }}><Typography>{t(kinds[item.kind] ?? 'processingUnknown')} · {state(item.status)}</Typography><Typography variant="body2"><time dateTime={item.atUtc}>{date(item.atUtc)}</time> · {stage(item.stage)} · {t('processingAttempts', { count: item.attempt, max: detail.maxAttempts })}</Typography>{item.failureCode ? <Typography variant="body2">{failure(item.failureCode)}</Typography> : null}</Box>)}</Box>}
|
||||
|
||||
Reference in New Issue
Block a user