cesnimda/jobtrackingapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(qa): qualify bounded revision timeline and notification volume

Browse Source
This commit is contained in:
cesnimda
2026-10-23 02:21:21 +02:00
parent 82be698dcf
commit 4b7727e22b
19 changed files with 9048 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/redesign/phase-9/evidence.py
+40 -10
View File
@@ -21,7 +21,7 @@ def browser_cases(suites):
rows.extend(browser_cases(suite.get('suites', [])))
for spec in suite.get('specs', []):
for test in spec.get('tests', []):
rows.append(dict(file=spec.get('file'), title=spec['title'], status=test['status'],
rows.append(dict(id=spec.get('id'), project=test.get('projectName'), file=spec.get('file'), title=spec['title'], status=test['status'],
results=[dict(status=r['status'], retry=r.get('retry', 0), durationMs=r.get('duration', 0),
errors=r.get('errors', []), attachments=r.get('attachments', [])) for r in test.get('results', [])]))
return rows
@@ -42,20 +42,45 @@ assert browser_cases([{'specs': [{'title': 'self-check', 'tests': [{'status': 'e
if __name__ == '__main__':
report = dict(observedUtc=datetime.now(timezone.utc).isoformat(), lanes={}, rawFiles={})
manifest_path = HERE / 'p9-00/source-manifest.json'
manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
changed = [name for name, expected in manifest.items() if not (ROOT / name).is_file() or digest(ROOT / name) != expected]
assert not changed, changed
report['sourceIdentity'] = dict(inputs=len(manifest), differences=changed, manifestSha256=digest(manifest_path),
qualification='Selected product/test, diagnostic, Release assembly and static-export bytes equal the frozen manifest. Documentation is outside this selected-source identity.')
report['documentOracle'] = dict(path='scripts/verify-cv-fidelity.py', sha256=digest(ROOT / 'scripts/verify-cv-fidelity.py'), qualification='Original unchanged oracle; native rendering and admission counts are separate.')
sources = dict(ordinary='ordinary-accepted', ai='ai', auth='auth', rate='rate', ingress='ingress', pdf='pdf', preferences='preferences', phase9='phase9-frozen')
report['supplementalSources'] = {str(path.relative_to(ROOT)): digest(path) for path in sorted((HERE / 'p9-05/volume').glob('*.ts'))}
discovered = json.loads((RAW / 'discovered.json').read_text(encoding='utf-8-sig'))
discovered_cases = browser_cases(discovered['suites'])
discovered_ids = {(case['id'], case['project']) for case in discovered_cases}
assert not discovered.get('errors') and len(discovered_cases) == len(discovered_ids) == 362
report['discovery'] = dict(cases=362, errors=discovered.get('errors', []), source='.artifacts/phase9/discovered.json',
qualification='List-only inventory; no tests executed. Its not-run statuses are not regression skip results.')
write_json(HERE / 'p9-00/discovered-cases.json', [{key: case[key] for key in ['id', 'project', 'file', 'title']} for case in discovered_cases])
sources = dict(ordinary=[f'ordinary-shard-{number}' for number in range(1, 5)], ai='ai', auth='auth', rate='rate', ingress='ingress', pdf='pdf', preferences='preferences', phase9='phase9-frozen', volume='volume-qualified')
for lane, source in sources.items():
file = RAW / f'{source}.json'
if not file.exists():
files = [RAW / f'{name}.json' for name in (source if isinstance(source, list) else [source])]
if not all(file.exists() for file in files):
continue
data = json.loads(file.read_text(encoding='utf-8-sig'))
cases = browser_cases(data['suites'])
report['lanes'][lane] = dict(stats=data['stats'], cases=len(cases),
datasets = [json.loads(file.read_text(encoding='utf-8-sig')) for file in files]
cases = [case for data in datasets for case in browser_cases(data['suites'])]
stats = dict(startTime=min(data['stats']['startTime'] for data in datasets),
**{key: sum(data['stats'][key] for data in datasets) for key in ['duration', 'expected', 'skipped', 'unexpected', 'flaky']})
if lane == 'ordinary':
assert len({(case['id'], case['project']) for case in cases}) == len(cases), 'Duplicate cases across native shards'
assert {(case['id'], case['project']) for case in cases} == discovered_ids, 'Shard union differs from full native discovery'
report['lanes'][lane] = dict(stats=stats, cases=len(cases),
retries=sum(max(len(c['results']) - 1, 0) for c in cases),
statuses={status: sum(c['status'] == status for c in cases) for status in ['expected', 'unexpected', 'flaky', 'skipped']},
fileFailures=data.get('errors', []))
folder = HERE / ('p9-01' if lane == 'ordinary' else 'p9-05' if lane == 'phase9' else 'p9-06')
fileFailures=[error for data in datasets for error in data.get('errors', [])],
sourceReports=[dict(file=str(file.relative_to(ROOT)), stats=data['stats']) for file, data in zip(files, datasets)])
if lane == 'ordinary':
report['lanes'][lane]['membershipMatchesDiscovery'] = True
folder = HERE / ('p9-01' if lane == 'ordinary' else 'p9-05' if lane in ['phase9', 'volume'] else 'p9-06')
folder.mkdir(parents=True, exist_ok=True)
if lane == 'volume':
observations = [dict(name=attachment['name'], observation=attached_json(attachment)) for case in cases for result in case['results'] for attachment in result['attachments'] if attachment['name'].startswith('phase9-volume-')]
write_json(HERE / 'p9-05/volume-results.json', observations)
if lane == 'phase9':
matrix, axe, screenshots = [], [], []
for case in cases:
@@ -84,6 +109,11 @@ if __name__ == '__main__':
for attachment in case['results'][0]['attachments']:
if attachment['name'].startswith('route-census-') and attachment['contentType'] == 'application/json':
write_json(HERE / 'p9-01' / (attachment['name'] + '.json'), attached_json(attachment))
elif attachment['name'] == 'actual-browser-zoom-observations':
observation = attached_json(attachment)
first = observation['records'][0]
assert first['language'] in ['en', 'nb'] and first['mode'] in ['light', 'dark']
write_json(HERE / 'p9-03' / f"zoom-{first['language']}-{first['mode']}.json", observation)
for case in cases:
for result in case['results']:
for attachment in result['attachments']:
@@ -105,7 +135,7 @@ if __name__ == '__main__':
if sidecar.exists():
report['lanes']['sidecar'] = ET.parse(sidecar).find('.//testsuite').attrib
report['failedOrSupersededAttempts'] = {}
for lane in ['phase9', 'phase9-successor', 'phase9-final', 'phase9-qualified', 'phase9-accepted', 'ordinary', 'ordinary-final', 'landing-red', 'landing-green', 'landing-qualified']:
for lane in ['phase9', 'phase9-successor', 'phase9-final', 'phase9-qualified', 'phase9-accepted', 'ordinary', 'ordinary-final', 'ordinary-accepted', 'ordinary-complete', 'volume', 'landing-red', 'landing-green', 'landing-qualified']:
file = RAW / f'{lane}.json'
if file.exists():
data = json.loads(file.read_text(encoding='utf-8-sig'))