test(handoff): add fail-closed evidence schema and qualification records
This commit is contained in:
cesnimda
@@ -0,0 +1,206 @@
|
||||
// Evidence validation only. Protected approvals and execution belong to the separate controller.
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import crypto from 'node:crypto';
|
||||
import zlib from 'node:zlib';
|
||||
import { createRequire } from 'node:module';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
|
||||
const directory = path.dirname(fileURLToPath(import.meta.url));
|
||||
const root = path.resolve(directory, '../../../..');
|
||||
const dependencies = createRequire(path.join(root, 'job-tracker-ui/package.json'));
|
||||
const Ajv = dependencies('ajv');
|
||||
const { JSDOM } = dependencies('jsdom');
|
||||
const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex');
|
||||
const demand = (condition, message) => { if (!condition) throw new Error(message); };
|
||||
|
||||
export function verifyBytes(bytes, expected) {
|
||||
demand(Buffer.isBuffer(bytes) && /^[a-f0-9]{64}$/.test(expected) && digest(bytes) === expected, 'Missing or changed evidence');
|
||||
}
|
||||
|
||||
export function measureNative(format, text) {
|
||||
demand(Buffer.byteLength(text, 'utf8') <= 8 * 1024 * 1024, 'Native report exceeds its input bound');
|
||||
const result = { passed: 0, failed: 0, skipped: 0, retries: 0, unexecuted: 0, names: [] };
|
||||
if (format === 'trx') {
|
||||
demand(!/<!DOCTYPE|<!ENTITY/i.test(text) && (text.match(/</g)?.length ?? 0) < 100000, 'XML declarations or excessive structure are not accepted');
|
||||
const document = new JSDOM(text, { contentType: 'text/xml' }).window.document;
|
||||
const counters = document.getElementsByTagNameNS('*', 'Counters')[0];
|
||||
demand(counters, 'Missing native TRX counters');
|
||||
for (const item of document.getElementsByTagNameNS('*', 'UnitTestResult')) {
|
||||
result.names.push(item.getAttribute('testName'));
|
||||
const outcome = item.getAttribute('outcome');
|
||||
if (outcome === 'Passed') result.passed++;
|
||||
else if (outcome === 'NotExecuted') result.skipped++;
|
||||
else result.failed++;
|
||||
}
|
||||
demand(Number(counters.getAttribute('total')) === result.names.length && Number(counters.getAttribute('passed')) === result.passed && Number(counters.getAttribute('failed')) === result.failed, 'TRX membership/counters disagree');
|
||||
} else if (format === 'junit') {
|
||||
demand(!/<!DOCTYPE|<!ENTITY/i.test(text) && (text.match(/</g)?.length ?? 0) < 100000, 'XML declarations or excessive structure are not accepted');
|
||||
const document = new JSDOM(text, { contentType: 'text/xml' }).window.document;
|
||||
for (const item of document.getElementsByTagName('testcase')) {
|
||||
result.names.push(`${item.getAttribute('classname')}::${item.getAttribute('name')}`);
|
||||
if (item.getElementsByTagName('skipped').length) result.skipped++;
|
||||
else if (item.getElementsByTagName('failure').length || item.getElementsByTagName('error').length) result.failed++;
|
||||
else result.passed++;
|
||||
}
|
||||
} else if (format === 'jest') {
|
||||
const report = JSON.parse(text);
|
||||
for (const suite of report.testResults ?? []) for (const item of suite.assertionResults ?? []) {
|
||||
const name = suite.name.replaceAll('\\', '/');
|
||||
const marker = name.lastIndexOf('/job-tracker-ui/');
|
||||
result.names.push(`${marker < 0 ? name : name.slice(marker + 1)}::${item.fullName}`);
|
||||
if (item.status === 'passed') result.passed++;
|
||||
else if (['pending', 'skipped', 'todo', 'disabled'].includes(item.status)) result.skipped++;
|
||||
else result.failed++;
|
||||
}
|
||||
demand(report.numTotalTests === result.names.length && report.numPassedTests === result.passed && report.numFailedTests === result.failed && report.numPendingTests === result.skipped && !report.numRuntimeErrorTestSuites, 'Jest membership/counters disagree');
|
||||
} else if (format === 'playwright') {
|
||||
const visit = suite => {
|
||||
for (const spec of suite.specs ?? []) for (const item of spec.tests ?? []) {
|
||||
result.names.push(`${spec.id}::${item.projectName}`);
|
||||
const attempts = item.results ?? [];
|
||||
result.retries += Math.max(0, attempts.length - 1);
|
||||
if (!attempts.length) result.unexecuted++;
|
||||
else if (attempts.at(-1).status === 'skipped') result.skipped++;
|
||||
else if (attempts.at(-1).status === 'passed' && item.expectedStatus !== 'failed' && item.status === 'expected') result.passed++;
|
||||
else result.failed++;
|
||||
if (attempts.some(attempt => (attempt.retry ?? 0) > 0)) result.retries = Math.max(1, result.retries);
|
||||
}
|
||||
for (const child of suite.suites ?? []) visit(child);
|
||||
};
|
||||
const report = JSON.parse(text);
|
||||
demand(!(report.errors?.length), 'Browser report contains execution errors');
|
||||
for (const suite of report.suites ?? []) visit(suite);
|
||||
} else if (format === 'pytest') {
|
||||
result.passed = Number(text.match(/\b(\d+) passed\b/)?.[1] ?? 0);
|
||||
result.failed = Number(text.match(/\b(\d+) failed\b/)?.[1] ?? 0);
|
||||
result.skipped = Number(text.match(/\b(\d+) skipped\b/)?.[1] ?? 0);
|
||||
} else if (format === 'unittest') {
|
||||
const count = Number(text.match(/Ran (\d+) tests? in/)?.[1] ?? 0);
|
||||
if (/^OK\s*$/m.test(text)) result.passed = count;
|
||||
else result.failed = count || 1;
|
||||
result.skipped = Number(text.match(/skipped=(\d+)/)?.[1] ?? 0);
|
||||
result.names = [...text.matchAll(/^(test\S+ \([^\n]+\)) \.\.\. /gm)].map(match => match[1]);
|
||||
} else if (format === 'node-test') {
|
||||
result.passed = Number(text.match(/(?:#|ℹ) pass (\d+)/u)?.[1] ?? 0);
|
||||
result.failed = Number(text.match(/(?:#|ℹ) fail (\d+)/u)?.[1] ?? 0);
|
||||
result.skipped = Number(text.match(/(?:#|ℹ) skipped (\d+)/u)?.[1] ?? 0);
|
||||
result.names = [...text.matchAll(/^(?:✔|✖|ok \d+ -|not ok \d+ -) (.+?)(?: \([\d.]+ms\))?$/gmu)].map(match => match[1]);
|
||||
} else throw new Error('Unknown native report format');
|
||||
result.names.sort();
|
||||
result.membershipSha256 = digest(Buffer.from(JSON.stringify(result.names)));
|
||||
return result;
|
||||
}
|
||||
|
||||
export function assertPass(result, exitCode) {
|
||||
demand(exitCode === 0 && result.passed > 0 && !result.failed && !result.skipped && !result.retries && !result.unexecuted, 'Native result cannot qualify as PASS');
|
||||
}
|
||||
|
||||
export function requireProtectedEntry() {
|
||||
return { eligible: false, reason: 'External protected phase10-development-entry, candidate scope, qualified runtime and test adapter are required. This evidence validator grants no authority.' };
|
||||
}
|
||||
|
||||
function readSafe(relative, maximum = 64 * 1024 * 1024, base = directory) {
|
||||
demand(typeof relative === 'string' && !path.isAbsolute(relative) && !relative.split(/[\\/]/).includes('..'), 'Unsafe artifact path');
|
||||
const file = path.resolve(base, relative);
|
||||
demand(fs.realpathSync(file).startsWith(fs.realpathSync(base) + path.sep), 'Artifact escapes package');
|
||||
const stat = fs.statSync(file);
|
||||
demand(stat.isFile() && stat.size <= maximum, 'Missing or excessive artifact');
|
||||
return fs.readFileSync(file);
|
||||
}
|
||||
|
||||
export function validatePackage(base = directory) {
|
||||
const readPackage = relative => readSafe(relative, 8 * 1024 * 1024, base);
|
||||
// The schema is trusted verifier source, never replaced by an imported package's schema.
|
||||
const schema = JSON.parse(readSafe('HANDOFF_SCHEMA.json'));
|
||||
verifyBytes(readPackage('HANDOFF_SCHEMA.json'), digest(readSafe('HANDOFF_SCHEMA.json')));
|
||||
const ajv = new Ajv({ allErrors: true, jsonPointers: true });
|
||||
const validate = ajv.compile(schema);
|
||||
const documents = {};
|
||||
for (const name of ['ACCEPTANCE', 'GATES', 'BLOCKERS', 'TEST_EVIDENCE']) {
|
||||
const document = JSON.parse(readPackage(name + '.json'));
|
||||
demand(validate(document), `Invalid ${name}: ${ajv.errorsText(validate.errors)}`);
|
||||
documents[name] = document;
|
||||
}
|
||||
const evidence = documents.TEST_EVIDENCE;
|
||||
demand(new Set(evidence.artifacts.map(item => item.id)).size === evidence.artifacts.length, 'Duplicate artifact IDs');
|
||||
demand(new Set(evidence.reports.map(item => item.id)).size === evidence.reports.length, 'Duplicate report IDs');
|
||||
for (const artifact of evidence.artifacts) verifyBytes(readPackage(artifact.path), artifact.sha256);
|
||||
const artifact = name => {
|
||||
const item = evidence.artifacts.find(entry => entry.id === name);
|
||||
demand(item, 'Missing indexed artifact');
|
||||
const bytes = readPackage(item.path);
|
||||
return item.path.endsWith('.gz') ? zlib.gunzipSync(bytes, { maxOutputLength: 8 * 1024 * 1024 }) : bytes;
|
||||
};
|
||||
for (const report of evidence.reports) {
|
||||
const expectedCommit = report.bindingScope === 'HANDOFF_VERIFIER' ? evidence.verifierCommit : evidence.testedCommit;
|
||||
demand(report.sourceCommit === expectedCommit, 'Report source correspondence missing');
|
||||
let nativeExit = JSON.parse(artifact(report.processArtifactId).toString('utf8'));
|
||||
for (const raw of report.processPointer.slice(1).split('/')) {
|
||||
const key = raw.replaceAll('~1', '/').replaceAll('~0', '~');
|
||||
demand(nativeExit !== null && Object.hasOwn(nativeExit, key), 'Missing native process observation');
|
||||
nativeExit = nativeExit[key];
|
||||
}
|
||||
demand(nativeExit === report.exitCode, 'Native process exit does not match the report');
|
||||
if (!report.nativeFormat) continue;
|
||||
const measured = measureNative(report.nativeFormat, artifact(report.artifactId).toString('utf8'));
|
||||
for (const key of ['passed', 'failed', 'skipped', 'retries', 'unexecuted']) demand(measured[key] === report.counts[key], `Wrong ${report.id} ${key}`);
|
||||
demand(measured.membershipSha256 === report.membershipSha256, 'Changed report membership');
|
||||
if (report.status === 'PASS') assertPass(measured, report.exitCode);
|
||||
}
|
||||
const discovery = measureNative('playwright', artifact(evidence.ordinaryDiscoveryArtifactId).toString('utf8')).names;
|
||||
const ordinary = evidence.ordinaryReportIds.flatMap(id => {
|
||||
const report = evidence.reports.find(item => item.id === id);
|
||||
demand(report?.nativeFormat === 'playwright' && report.status === 'PASS', 'Missing ordinary browser lane');
|
||||
return measureNative('playwright', artifact(report.artifactId).toString('utf8')).names;
|
||||
}).sort();
|
||||
demand(new Set(ordinary).size === ordinary.length && JSON.stringify(ordinary) === JSON.stringify(discovery), 'Browser discovery differs from executed unique membership');
|
||||
const source = JSON.parse(artifact(evidence.sourceManifestArtifactId).toString('utf8'));
|
||||
const after = JSON.parse(artifact(evidence.sourceAfterArtifactId).toString('utf8'));
|
||||
demand(JSON.stringify(source) === JSON.stringify(after), 'Source drifted during the regression run');
|
||||
const prefixes = ['JobTrackerApi/', 'JobTrackerApi.Tests/', 'JobTrackerApi.MariaDb.Tests/', 'job-tracker-ui/', 'scripts/', 'tools/summarizer/', 'docs/pre-phase10-ux/', 'docs/redesign/phase-9/'];
|
||||
const tracked = execFileSync('rtk', ['proxy', 'git', 'ls-files', '-z'], { cwd: root, encoding: 'utf8' }).split('\0').filter(name => prefixes.some(prefix => name.startsWith(prefix))).sort();
|
||||
demand(Object.keys(source).length <= 5000 && JSON.stringify(Object.keys(source).sort()) === JSON.stringify(tracked), 'Frozen source membership does not match the tracked qualification scope');
|
||||
for (const [name, expected] of Object.entries(source)) {
|
||||
demand(!path.isAbsolute(name) && !name.split(/[\\/]/).includes('..'), 'Unsafe source path');
|
||||
const file = path.resolve(root, name);
|
||||
demand(fs.realpathSync(file).startsWith(fs.realpathSync(root) + path.sep), 'Source escapes repository');
|
||||
verifyBytes(fs.readFileSync(file), expected);
|
||||
}
|
||||
const verifierFiles = JSON.parse(artifact(evidence.verifierSourceArtifactId).toString('utf8'));
|
||||
const expectedVerifierFiles = ['verify.mjs', 'verify.test.mjs', 'package.test.mjs', 'verify_git.py', 'HANDOFF_SCHEMA.json'].map(name => 'docs/automation/handoffs/pre-phase10/' + name).sort();
|
||||
demand(JSON.stringify(Object.keys(verifierFiles).sort()) === JSON.stringify(expectedVerifierFiles), 'Missing verifier source membership');
|
||||
for (const [name, expected] of Object.entries(verifierFiles)) verifyBytes(fs.readFileSync(path.join(root, name)), expected);
|
||||
execFileSync('rtk', ['proxy', 'git', 'merge-base', '--is-ancestor', evidence.verifierCommit, 'HEAD'], { cwd: root });
|
||||
execFileSync('rtk', ['proxy', 'git', 'merge-base', '--is-ancestor', evidence.testedCommit, 'HEAD'], { cwd: root });
|
||||
const delta = execFileSync('rtk', ['proxy', 'git', 'diff', '--name-only', evidence.testedCommit, 'HEAD'], { cwd: root, encoding: 'utf8' }).trim().split(/\r?\n/).filter(Boolean);
|
||||
demand(delta.every(name => name.startsWith('docs/automation/handoffs/pre-phase10/')), 'Application changed after tested baseline');
|
||||
const declared = documents.ACCEPTANCE;
|
||||
demand(declared.localCommit === evidence.testedCommit && declared.authority === 'EVIDENCE_ONLY' && declared.phase10DevelopmentReadiness === 'NO-GO' && declared.eligibleForAutomaticExecution === false, 'Unapproved entry claim');
|
||||
const ids = new Set(documents.BLOCKERS.items.map(item => item.id));
|
||||
demand(ids.size === documents.BLOCKERS.items.length, 'Duplicate blocker IDs');
|
||||
demand(declared.unresolvedBlockerIds.every(id => ids.has(id)), 'Unresolved blocker is not registered');
|
||||
const observedLocalCommit = execFileSync('rtk', ['proxy', 'git', 'rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).trim();
|
||||
const observedWorkingTreeClean = !execFileSync('rtk', ['proxy', 'git', 'status', '--porcelain'], { cwd: root, encoding: 'utf8' }).trim();
|
||||
const branch = execFileSync('rtk', ['proxy', 'git', 'branch', '--show-current'], { cwd: root, encoding: 'utf8' }).trim();
|
||||
demand(branch === declared.sourceBranch, 'Wrong checkout branch');
|
||||
return { integrity: 'PASS', testedCommit: evidence.testedCommit, observedLocalCommit, observedWorkingTreeClean, reports: evidence.reports.length, artifacts: evidence.artifacts.length, phase10: 'BLOCKED', ...requireProtectedEntry() };
|
||||
}
|
||||
|
||||
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
||||
try {
|
||||
const result = validatePackage();
|
||||
if (process.argv.includes('--verify-remote')) {
|
||||
const branch = 'refs/heads/test/pre-phase10-ux-acceptance';
|
||||
const line = execFileSync('rtk', ['proxy', 'git', 'ls-remote', '--exit-code', 'origin', branch], { cwd: root, encoding: 'utf8', timeout: 45000 }).trim();
|
||||
demand(line === `${result.observedLocalCommit}\t${branch}`, 'Remote HEAD differs from current local HEAD');
|
||||
result.currentRemotePublication = 'VERIFIED';
|
||||
} else result.currentRemotePublication = 'NOT_RECHECKED';
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
if (process.argv.includes('--require-ready')) process.exitCode = 3;
|
||||
} catch (error) {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user