"""Summarize completed raw QA artifacts; missing lanes stay missing.""" import hashlib import base64 import json from pathlib import Path import xml.etree.ElementTree as ET from datetime import datetime, timezone ROOT = Path(__file__).resolve().parents[3] HERE = Path(__file__).resolve().parent RAW = ROOT / '.artifacts/phase9' def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() def browser_cases(suites): rows = [] for suite in suites: rows.extend(browser_cases(suite.get('suites', []))) for spec in suite.get('specs', []): for test in spec.get('tests', []): rows.append(dict(id=spec.get('id'), project=test.get('projectName'), file=spec.get('file'), title=spec['title'], status=test['status'], results=[dict(status=r['status'], retry=r.get('retry', 0), durationMs=r.get('duration', 0), errors=r.get('errors', []), attachments=r.get('attachments', [])) for r in test.get('results', [])])) return rows def attached_json(attachment): body = base64.b64decode(attachment['body']) if 'body' in attachment else Path(attachment['path']).read_bytes() return json.loads(body) def write_json(path, value): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') assert browser_cases([{'specs': [{'title': 'self-check', 'tests': [{'status': 'expected', 'results': [{'status': 'passed'}]}]}]}])[0]['results'][0]['status'] == 'passed' if __name__ == '__main__': report = dict(observedUtc=datetime.now(timezone.utc).isoformat(), lanes={}, rawFiles={}) manifest_path = HERE / 'p9-00/source-manifest.json' manifest = json.loads(manifest_path.read_text(encoding='utf-8')) changed = [name for name, expected in manifest.items() if not (ROOT / name).is_file() or digest(ROOT / name) != expected] assert not changed, changed report['sourceIdentity'] = dict(inputs=len(manifest), differences=changed, manifestSha256=digest(manifest_path), qualification='Selected product/test, diagnostic, Release assembly and static-export bytes equal the frozen manifest. Documentation is outside this selected-source identity.') report['documentOracle'] = dict(path='scripts/verify-cv-fidelity.py', sha256=digest(ROOT / 'scripts/verify-cv-fidelity.py'), qualification='Original unchanged oracle; native rendering and admission counts are separate.') report['supplementalSources'] = {str(path.relative_to(ROOT)): digest(path) for path in sorted((HERE / 'p9-05/volume').glob('*.ts'))} discovered = json.loads((RAW / 'discovered.json').read_text(encoding='utf-8-sig')) discovered_cases = browser_cases(discovered['suites']) discovered_ids = {(case['id'], case['project']) for case in discovered_cases} assert not discovered.get('errors') and len(discovered_cases) == len(discovered_ids) == 362 report['discovery'] = dict(cases=362, errors=discovered.get('errors', []), source='.artifacts/phase9/discovered.json', qualification='List-only inventory; no tests executed. Its not-run statuses are not regression skip results.') write_json(HERE / 'p9-00/discovered-cases.json', [{key: case[key] for key in ['id', 'project', 'file', 'title']} for case in discovered_cases]) sources = dict(ordinary=[f'ordinary-shard-{number}' for number in range(1, 5)], ai='ai', auth='auth', rate='rate', ingress='ingress', pdf='pdf', preferences='preferences', phase9='phase9-frozen', volume='volume-qualified') for lane, source in sources.items(): files = [RAW / f'{name}.json' for name in (source if isinstance(source, list) else [source])] if not all(file.exists() for file in files): continue datasets = [json.loads(file.read_text(encoding='utf-8-sig')) for file in files] cases = [case for data in datasets for case in browser_cases(data['suites'])] stats = dict(startTime=min(data['stats']['startTime'] for data in datasets), **{key: sum(data['stats'][key] for data in datasets) for key in ['duration', 'expected', 'skipped', 'unexpected', 'flaky']}) if lane == 'ordinary': assert len({(case['id'], case['project']) for case in cases}) == len(cases), 'Duplicate cases across native shards' assert {(case['id'], case['project']) for case in cases} == discovered_ids, 'Shard union differs from full native discovery' report['lanes'][lane] = dict(stats=stats, cases=len(cases), retries=sum(max(len(c['results']) - 1, 0) for c in cases), statuses={status: sum(c['status'] == status for c in cases) for status in ['expected', 'unexpected', 'flaky', 'skipped']}, fileFailures=[error for data in datasets for error in data.get('errors', [])], sourceReports=[dict(file=str(file.relative_to(ROOT)), stats=data['stats']) for file, data in zip(files, datasets)]) if lane == 'ordinary': report['lanes'][lane]['membershipMatchesDiscovery'] = True folder = HERE / ('p9-01' if lane == 'ordinary' else 'p9-05' if lane in ['phase9', 'volume'] else 'p9-06') folder.mkdir(parents=True, exist_ok=True) if lane == 'volume': observations = [dict(name=attachment['name'], observation=attached_json(attachment)) for case in cases for result in case['results'] for attachment in result['attachments'] if attachment['name'].startswith('phase9-volume-')] write_json(HERE / 'p9-05/volume-results.json', observations) if lane == 'phase9': matrix, axe, screenshots = [], [], [] for case in cases: for attachment in case['results'][0]['attachments']: name = attachment['name'] if name == 'phase9-route-matrix': sample = attached_json(attachment) for row in sample['samples']: row.pop('bodyText', None) matrix.append(row) elif name == 'phase9-settled-axe': axe.append(attached_json(attachment)) elif name == 'phase9-rd21-characterization': write_json(HERE / 'p9-04/rd21.json', attached_json(attachment)) elif name == 'phase9-sustained-performance': write_json(HERE / 'p9-05/performance.json', attached_json(attachment)) if attachment['contentType'] == 'image/png': path = Path(attachment['path']) screenshots.append(dict(case=case['title'], name=name, path=str(path.relative_to(ROOT)), sha256=digest(path))) assert len(matrix) == 396 write_json(HERE / 'p9-02/route-results.json', matrix) write_json(HERE / 'p9-02/screenshots.json', screenshots) write_json(HERE / 'p9-03/settled-axe.json', axe) if lane == 'ordinary': for case in cases: for attachment in case['results'][0]['attachments']: if attachment['name'].startswith('route-census-') and attachment['contentType'] == 'application/json': write_json(HERE / 'p9-01' / (attachment['name'] + '.json'), attached_json(attachment)) elif attachment['name'] == 'actual-browser-zoom-observations': observation = attached_json(attachment) first = observation['records'][0] assert first['language'] in ['en', 'nb'] and first['mode'] in ['light', 'dark'] write_json(HERE / 'p9-03' / f"zoom-{first['language']}-{first['mode']}.json", observation) for case in cases: for result in case['results']: for attachment in result['attachments']: if 'body' in attachment: body = base64.b64decode(attachment.pop('body')) attachment.update(bytes=len(body), sha256=hashlib.sha256(body).hexdigest()) (folder / f'{lane}-cases.json').write_text(json.dumps(cases, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') frontend = RAW / 'frontend.json' if frontend.exists(): data = json.loads(frontend.read_text(encoding='utf-8-sig')) report['lanes']['frontend'] = {k: data.get(k) for k in ['success', 'numPassedTestSuites', 'numFailedTestSuites', 'numPassedTests', 'numFailedTests', 'numPendingTests', 'numTodoTests', 'startTime']} backend = RAW / 'backend/phase9.trx' if backend.exists(): doc = ET.parse(backend) report['lanes']['backend'] = doc.find('.//{*}Counters').attrib cases = [dict(name=e.get('testName'), outcome=e.get('outcome'), duration=e.get('duration')) for e in doc.findall('.//{*}UnitTestResult')] (HERE / 'p9-00/backend-cases.json').write_text(json.dumps(cases, indent=2) + '\n', encoding='utf-8') sidecar = RAW / 'sidecar.xml' if sidecar.exists(): report['lanes']['sidecar'] = ET.parse(sidecar).find('.//testsuite').attrib report['failedOrSupersededAttempts'] = {} for lane in ['phase9', 'phase9-successor', 'phase9-final', 'phase9-qualified', 'phase9-accepted', 'ordinary', 'ordinary-final', 'ordinary-accepted', 'ordinary-complete', 'volume', 'landing-red', 'landing-green', 'landing-qualified']: file = RAW / f'{lane}.json' if file.exists(): data = json.loads(file.read_text(encoding='utf-8-sig')) report['failedOrSupersededAttempts'][lane] = data.get('stats') for file in sorted(RAW.glob('*-exit.json')): report['lanes'][file.stem.replace('-exit', '') + 'Process'] = json.loads(file.read_text(encoding='utf-8-sig')) for file in sorted(RAW.rglob('*')): if file.is_file() and file.suffix in ['.json', '.xml', '.trx', '.log'] and 'results' not in file.parts: report['rawFiles'][str(file.relative_to(ROOT))] = dict(bytes=file.stat().st_size, sha256=digest(file)) (HERE / 'validation.json').write_text(json.dumps(report, indent=2, ensure_ascii=False) + '\n', encoding='utf-8') print(json.dumps(report['lanes'], indent=2))