cesnimda
83 lines
5.0 KiB
Python
83 lines
5.0 KiB
Python
"""Offline fail-closed checks for the requalification result ledger."""
|
|
import copy
|
|
import importlib.util
|
|
from pathlib import Path
|
|
import unittest
|
|
import xml.etree.ElementTree as ET
|
|
|
|
spec = importlib.util.spec_from_file_location('requalification', Path(__file__).with_name('reconcile.py'))
|
|
ledger = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(ledger)
|
|
|
|
def report(identifier='one'):
|
|
return {'errors': [], 'suites': [{'specs': [{'id': identifier, 'title': identifier,
|
|
'tests': [{'projectName': 'chromium', 'expectedStatus': 'passed',
|
|
'results': [{'status': 'passed', 'retry': 0}]}]}]}]}
|
|
|
|
class EvidenceGuards(unittest.TestCase):
|
|
def test_accepts_completed_native_case(self):
|
|
self.assertEqual(len(ledger.browser_results(report(), 0, 1)), 1)
|
|
|
|
def test_failed_process_cannot_be_qualified_by_passing_cases(self):
|
|
with self.assertRaises(ValueError): ledger.browser_results(report(), 1, 1)
|
|
|
|
def test_rejects_skips_retries_and_missing_results(self):
|
|
for results in [[{'status':'skipped','retry':0}], [{'status':'passed','retry':1}], []]:
|
|
value=report(); value['suites'][0]['specs'][0]['tests'][0]['results']=results
|
|
with self.assertRaises(ValueError): ledger.browser_results(value, 0, 1)
|
|
|
|
def test_rejects_expected_failure_and_run_errors(self):
|
|
value=report(); value['suites'][0]['specs'][0]['tests'][0]['expectedStatus']='failed'
|
|
with self.assertRaises(ValueError): ledger.browser_results(value, 0, 1)
|
|
value=report(); value['errors']=[{'message':'web server failed'}]
|
|
with self.assertRaises(ValueError): ledger.browser_results(value, 0, 1)
|
|
|
|
def test_rejects_count_mismatch_and_duplicate_cases(self):
|
|
with self.assertRaises(ValueError): ledger.browser_results(report(), 0, 2)
|
|
value=report(); value['suites']*=2
|
|
with self.assertRaises(ValueError): ledger.browser_results(value, 0)
|
|
|
|
def test_discovery_only_skip_status_is_not_runtime_skip_evidence(self):
|
|
discovery=report(); discovery['suites'][0]['specs'][0]['tests'][0]['results']=[]
|
|
discovery['stats']={'skipped':1,'expected':0}
|
|
self.assertEqual(len(ledger.reconcile_ordinary(discovery, [ledger.browser_results(report(),0)])),1)
|
|
|
|
def test_shards_cannot_overlap_or_omit_discovery_members(self):
|
|
cases=ledger.browser_results(report(),0)
|
|
with self.assertRaises(ValueError): ledger.reconcile_ordinary(report(), [cases,cases])
|
|
with self.assertRaises(ValueError): ledger.reconcile_ordinary(report('different'), [cases])
|
|
|
|
def test_final_gate_cannot_succeed_with_incomplete_or_failed_lanes(self):
|
|
ledger.require_complete({'one': {'status': 'PASS'}})
|
|
for status in ['FAIL', 'NOT_RUN', 'NOT_COMPLETE', 'BLOCKED', 'FINDINGS', 'INCOMPLETE']:
|
|
with self.assertRaises(ValueError): ledger.require_complete({'one': {'status':'PASS'}, 'two': {'status':status}})
|
|
|
|
def test_sidecar_requires_unique_successful_native_cases(self):
|
|
value=ET.fromstring('<testsuites><testsuite><testcase classname="a" name="one"/><testcase classname="a" name="two"/></testsuite></testsuites>')
|
|
self.assertEqual(len(ledger.sidecar_results(value,2)),2)
|
|
duplicate=ET.fromstring('<testsuites><testsuite><testcase classname="a" name="one"/><testcase classname="a" name="one"/></testsuite></testsuites>')
|
|
with self.assertRaises(ValueError): ledger.sidecar_results(duplicate,2)
|
|
failed=ET.fromstring('<testsuites><testsuite><testcase classname="a" name="one"><skipped/></testcase></testsuite></testsuites>')
|
|
with self.assertRaises(ValueError): ledger.sidecar_results(failed,1)
|
|
|
|
def test_jest_parameterized_labels_require_exact_baseline_multiset(self):
|
|
value={'testResults':[{'name':'suite.ts','assertionResults':[
|
|
{'fullName':'same label','status':'passed','invocations':1},
|
|
{'fullName':'same label','status':'passed','invocations':1}]}]}
|
|
self.assertEqual(len(ledger.jest_results(value,value,0,2)),2)
|
|
extra=copy.deepcopy(value); extra['testResults'][0]['assertionResults'].append(extra['testResults'][0]['assertionResults'][0])
|
|
with self.assertRaises(ValueError): ledger.jest_results(extra,value,0,3)
|
|
retried=copy.deepcopy(value); retried['testResults'][0]['assertionResults'][0]['invocations']=2
|
|
with self.assertRaises(ValueError): ledger.jest_results(retried,value,0,2)
|
|
|
|
def test_security_rejects_missing_commands_and_empty_project_evidence(self):
|
|
with self.assertRaises(ValueError): ledger.security_results([],{'metadata':{'vulnerabilities':{'total':0}}},{'projects':[]})
|
|
|
|
def test_known_python_advisories_cannot_be_a_clean_security_gate(self):
|
|
value={'advisoryGroups':[{'identifiers':['one']}],'errors':[{'httpStatus':404}]}
|
|
self.assertEqual(ledger.python_security_status(value),'FINDINGS')
|
|
self.assertEqual(ledger.python_security_status({'advisoryGroups':[],'errors':[{}]}),'INCOMPLETE')
|
|
self.assertEqual(ledger.python_security_status({'advisoryGroups':[],'errors':[]}),'PASS')
|
|
|
|
if __name__ == '__main__': unittest.main()
|