"""Offline fail-closed checks for the requalification result ledger.""" import copy import importlib.util from pathlib import Path import unittest import xml.etree.ElementTree as ET spec = importlib.util.spec_from_file_location('requalification', Path(__file__).with_name('reconcile.py')) ledger = importlib.util.module_from_spec(spec) spec.loader.exec_module(ledger) def report(identifier='one'): return {'errors': [], 'suites': [{'specs': [{'id': identifier, 'title': identifier, 'tests': [{'projectName': 'chromium', 'expectedStatus': 'passed', 'results': [{'status': 'passed', 'retry': 0}]}]}]}]} class EvidenceGuards(unittest.TestCase): def test_accepts_completed_native_case(self): self.assertEqual(len(ledger.browser_results(report(), 0, 1)), 1) def test_failed_process_cannot_be_qualified_by_passing_cases(self): with self.assertRaises(ValueError): ledger.browser_results(report(), 1, 1) def test_rejects_skips_retries_and_missing_results(self): for results in [[{'status':'skipped','retry':0}], [{'status':'passed','retry':1}], []]: value=report(); value['suites'][0]['specs'][0]['tests'][0]['results']=results with self.assertRaises(ValueError): ledger.browser_results(value, 0, 1) def test_rejects_expected_failure_and_run_errors(self): value=report(); value['suites'][0]['specs'][0]['tests'][0]['expectedStatus']='failed' with self.assertRaises(ValueError): ledger.browser_results(value, 0, 1) value=report(); value['errors']=[{'message':'web server failed'}] with self.assertRaises(ValueError): ledger.browser_results(value, 0, 1) def test_rejects_count_mismatch_and_duplicate_cases(self): with self.assertRaises(ValueError): ledger.browser_results(report(), 0, 2) value=report(); value['suites']*=2 with self.assertRaises(ValueError): ledger.browser_results(value, 0) def test_discovery_only_skip_status_is_not_runtime_skip_evidence(self): discovery=report(); discovery['suites'][0]['specs'][0]['tests'][0]['results']=[] discovery['stats']={'skipped':1,'expected':0} self.assertEqual(len(ledger.reconcile_ordinary(discovery, [ledger.browser_results(report(),0)])),1) def test_shards_cannot_overlap_or_omit_discovery_members(self): cases=ledger.browser_results(report(),0) with self.assertRaises(ValueError): ledger.reconcile_ordinary(report(), [cases,cases]) with self.assertRaises(ValueError): ledger.reconcile_ordinary(report('different'), [cases]) def test_final_gate_cannot_succeed_with_incomplete_or_failed_lanes(self): ledger.require_complete({'one': {'status': 'PASS'}}) for status in ['FAIL', 'NOT_RUN', 'NOT_COMPLETE', 'BLOCKED', 'FINDINGS', 'INCOMPLETE']: with self.assertRaises(ValueError): ledger.require_complete({'one': {'status':'PASS'}, 'two': {'status':status}}) def test_sidecar_requires_unique_successful_native_cases(self): value=ET.fromstring('') self.assertEqual(len(ledger.sidecar_results(value,2)),2) duplicate=ET.fromstring('') with self.assertRaises(ValueError): ledger.sidecar_results(duplicate,2) failed=ET.fromstring('') with self.assertRaises(ValueError): ledger.sidecar_results(failed,1) def test_jest_parameterized_labels_require_exact_baseline_multiset(self): value={'testResults':[{'name':'suite.ts','assertionResults':[ {'fullName':'same label','status':'passed','invocations':1}, {'fullName':'same label','status':'passed','invocations':1}]}]} self.assertEqual(len(ledger.jest_results(value,value,0,2)),2) extra=copy.deepcopy(value); extra['testResults'][0]['assertionResults'].append(extra['testResults'][0]['assertionResults'][0]) with self.assertRaises(ValueError): ledger.jest_results(extra,value,0,3) retried=copy.deepcopy(value); retried['testResults'][0]['assertionResults'][0]['invocations']=2 with self.assertRaises(ValueError): ledger.jest_results(retried,value,0,2) def test_security_rejects_missing_commands_and_empty_project_evidence(self): with self.assertRaises(ValueError): ledger.security_results([],{'metadata':{'vulnerabilities':{'total':0}}},{'projects':[]}) def test_known_python_advisories_cannot_be_a_clean_security_gate(self): value={'advisoryGroups':[{'identifiers':['one']}],'errors':[{'httpStatus':404}]} self.assertEqual(ledger.python_security_status(value),'FINDINGS') self.assertEqual(ledger.python_security_status({'advisoryGroups':[],'errors':[{}]}),'INCOMPLETE') self.assertEqual(ledger.python_security_status({'advisoryGroups':[],'errors':[]}),'PASS') if __name__ == '__main__': unittest.main()