cesnimda
259 lines
21 KiB
Python
259 lines
21 KiB
Python
"""Source-bound successor accounting; the one declared frontend addition never masks missing cases."""
|
|
import copy
|
|
import datetime
|
|
import importlib.util
|
|
import hashlib
|
|
import json
|
|
import sys
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
HERE=Path(__file__).resolve().parent
|
|
spec=importlib.util.spec_from_file_location('qualified_reconciliation',HERE.parent/'p9-requal-01/reconcile.py')
|
|
prior=importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(prior)
|
|
ROOT=prior.ROOT
|
|
RAW=ROOT/'.artifacts/p9-sec-02/final'
|
|
prior.RAW=RAW
|
|
ADDED_CASE='focusing a preview retry control keeps its pointer target stationary'
|
|
|
|
|
|
def require_fresh(path):
|
|
if not Path(path).resolve().is_relative_to((ROOT/'.artifacts/p9-sec-02').resolve()):
|
|
raise ValueError('Fresh execution cannot use a historical evidence directory.')
|
|
|
|
def validate_representative(proof,report):
|
|
expected={'long-skills-minimal-en','long-skills-technical-nb'}
|
|
cases=report.get('fixtures',[])
|
|
if (proof.get('status')!='PASS' or proof.get('oracleExitCode')!=0
|
|
or report.get('total')!=2 or report.get('passed')!=2 or len(cases)!=2
|
|
or {case.get('fixture') for case in cases}!=expected
|
|
or any(case.get('pass') is not True or case.get('failures')!=[] for case in cases)
|
|
or proof.get('cases')!=cases):
|
|
raise ValueError('Supplemental PDF membership, oracle outcome or retained cases differ.')
|
|
|
|
def require_unchanged(inputs):
|
|
if any(not Path(name).is_file() or hashlib.sha256(Path(name).read_bytes()).hexdigest()!=digest for name,digest in inputs.items()):
|
|
raise ValueError('Supplemental PDF input/report drift.')
|
|
|
|
def frontend_cases(report,baseline,exit_code):
|
|
reference=copy.deepcopy(baseline)
|
|
suites=[suite for suite in reference['testResults'] if suite['name'].replace('\\','/').endswith('/job-tracker-ui/src/cv-builder-deep-link.test.tsx')]
|
|
if len(suites)!=1: raise ValueError('Missing or ambiguous frontend membership reference.')
|
|
# This adds an expectation, never an execution outcome, to the old934-case multiset.
|
|
suites[0]['assertionResults'].append({'fullName':ADDED_CASE})
|
|
return prior.jest_results(report,reference,exit_code,935)
|
|
|
|
def main():
|
|
results={'backend':prior.collect_trx('backend','backend/backend.trx',1130),
|
|
'mariadb':prior.collect_trx('mariadb','mariadb/mariadb.trx',35)}
|
|
for i in range(1,5): results[f'ordinary-{i}']=prior.collect_browser(f'ordinary-{i}')
|
|
amendment=None
|
|
successor=ROOT/'.artifacts/p9-sec-02/oauth-fixture-successor'
|
|
if (successor/'result.json').exists():
|
|
predecessor=results['ordinary-3']
|
|
try:
|
|
predecessor['nativeStats']=prior.load(RAW/'ordinary-3.json').get('stats')
|
|
predecessor['report']=prior.identity(RAW/'ordinary-3.json')
|
|
amendment=prior.load(successor/'result.json')
|
|
if amendment.get('status')!='PASS' or amendment.get('exitCode')!=0: raise ValueError('Fixture successor did not pass.')
|
|
if amendment['file']!='job-tracker-ui/e2e/phase7-oauth-sender.spec.ts': raise ValueError('Unexpected amended fixture owner.')
|
|
if prior.identity(ROOT/amendment['file'])['sha256']!=amendment['after']: raise ValueError('Amended fixture source drift.')
|
|
old_discovery=prior.load(RAW/'discovery.json'); new_discovery=prior.load(successor/'discovery.json')
|
|
ids=lambda d: {s['id']+':'+t['projectName'] for s in prior.specs(d) for t in s['tests']}
|
|
if ids(old_discovery)!=ids(new_discovery) or new_discovery.get('errors'): raise ValueError('Amended fixture changed ordinary discovery.')
|
|
prior.RAW=successor
|
|
current=prior.collect_browser('ordinary-3',88)
|
|
prior.RAW=RAW
|
|
if current['status']!='PASS': raise ValueError('Complete88-case successor missing.')
|
|
current['supersededAttempt']=predecessor
|
|
current['amendment']=amendment
|
|
results['ordinary-3']=current
|
|
except (ValueError,KeyError) as error:
|
|
results['ordinary-3']={'status':'FAIL','reason':str(error),'supersededAttempt':predecessor}
|
|
finally:prior.RAW=RAW
|
|
try:
|
|
if any(results[f'ordinary-{i}']['status']!='PASS' for i in range(1,5)): raise ValueError('Incomplete ordinary shards.')
|
|
cases=prior.reconcile_ordinary(prior.load(RAW/'discovery.json'),[results[f'ordinary-{i}']['cases'] for i in range(1,5)])
|
|
if len(cases)!=362: raise ValueError('Unexpected ordinary discovery membership.')
|
|
results['ordinary']={'status':'PASS','passed':362,'discovery':prior.identity(RAW/'discovery.json')}
|
|
except (FileNotFoundError,ValueError) as error: results['ordinary']={'status':'NOT_COMPLETE','reason':str(error)}
|
|
for name,count in {'ai':2,'auth':2,'rate':9,'ingress':6,'pdf-admission':1,'preferences':3,'rd21':6,'diagnostics':18,'volume':3,'rd21-volume':3,'mariadb-browser':1,'mariadb-observations':2}.items():
|
|
results[name]=prior.collect_browser(name,count)
|
|
old_raw=prior.RAW
|
|
prior.RAW=ROOT/'.artifacts/p9-sec-02/final-linux-02'
|
|
results['linux-renderer']=prior.collect_browser('linux-renderer',5)
|
|
results['linux-renderer']['supersededAttempt']={'status':'FAIL','executedBrowserCases':0,
|
|
'reason':'New readiness label rejected by unchanged historical fixture guard before case execution; original guard token retained in source/image-verified successor.',
|
|
'process':prior.identity(ROOT/'.artifacts/p9-sec-02/final-linux/linux/process.json')}
|
|
prior.RAW=ROOT/'.artifacts/p9-sec-02/representative-linux'
|
|
representative=prior.collect_browser('linux-renderer',2)
|
|
try:
|
|
proof=prior.load(prior.RAW/'result.json')
|
|
root=prior.RAW/'linux/fidelity'
|
|
report=prior.load(root/'fidelity-results.json')
|
|
validate_representative(proof,report)
|
|
measurements=prior.load(root/'measurements.json')
|
|
if len(measurements['fixtures'])!=2 or {case['fixture'] for case in measurements['fixtures']}!={case['fixture'] for case in report['fixtures']}:
|
|
raise ValueError('Supplemental measurements membership differs.')
|
|
binding=prior.RAW/'oracle-inputs.json'
|
|
inputs=prior.load(binding)['inputs']
|
|
if proof.get('inputs')!=inputs or proof.get('reportSha256')!=hashlib.sha256((root/'fidelity-results.json').read_bytes()).hexdigest():
|
|
raise ValueError('Supplemental producer input/report binding differs.')
|
|
oracle=str((ROOT/'scripts/verify-cv-fidelity.py').resolve())
|
|
if inputs.get(oracle)!='64f82e622cc11483c33307d987e969f6cf27f82ab6bdbd72c2aeb1bfbceade87':
|
|
raise ValueError('Supplemental original oracle identity differs.')
|
|
for case in report['fixtures']:
|
|
for name in ['render.html','export.pdf','measurements.json']:
|
|
if str((root/case['fixture']/name).resolve()) not in inputs: raise ValueError('Supplemental required artifact missing.')
|
|
require_unchanged(inputs)
|
|
representative['oracle']=proof
|
|
representative['oracleEvidence']=prior.identity(prior.RAW/'result.json')
|
|
representative['inputBinding']=prior.identity(binding)
|
|
representative['report']=prior.identity(root/'fidelity-results.json')
|
|
except FileNotFoundError as error: representative.update(status='NOT_COMPLETE',reason=str(error))
|
|
except ValueError as error: representative.update(status='FAIL',reason=str(error))
|
|
results['pdf-representative']=representative
|
|
prior.RAW=old_raw
|
|
try:
|
|
path=RAW/'frontend.json';data=prior.load(path);process=prior.load(RAW/'frontend-exit.json')
|
|
reference=ROOT/'.artifacts/p9-requal-01/frontend.json'
|
|
accepted=prior.load(HERE.parent/'p9-requal-01/validation.json')['results']['frontend']['report']['sha256']
|
|
if prior.identity(reference)['sha256']!=accepted: raise ValueError('Historical frontend reference changed.')
|
|
if f"test('{ADDED_CASE}'" not in (ROOT/'job-tracker-ui/src/cv-builder-deep-link.test.tsx').read_text(encoding='utf-8'): raise ValueError('Declared added regression is absent from source.')
|
|
cases=frontend_cases(data,prior.load(reference),process['exitCode'])
|
|
expected={'numPassedTests':935,'numTotalTests':935,'numFailedTests':0,'numPendingTests':0,'numTodoTests':0,'numPassedTestSuites':105,'numTotalTestSuites':105,'numFailedTestSuites':0}
|
|
if any(data.get(k)!=v for k,v in expected.items()): raise ValueError('Frontend aggregate counts differ.')
|
|
results['frontend']={'status':'PASS','passed':935,'suites':105,'cases':cases,'report':prior.identity(path),'process':process,'membershipReference':prior.identity(reference),'declaredAddedCase':ADDED_CASE}
|
|
except FileNotFoundError as error: results['frontend']={'status':'NOT_RUN','reason':str(error)}
|
|
except ValueError as error: results['frontend']={'status':'FAIL','reason':str(error)}
|
|
for name in ['types','lint','build']:
|
|
try:
|
|
process=prior.load(RAW/(name+'-exit.json'))
|
|
result={'status':'PASS' if process['exitCode']==0 else 'FAIL','process':process,'log':prior.identity(RAW/(name+'.log'))}
|
|
results[name]=result
|
|
except FileNotFoundError as error: results[name]={'status':'NOT_RUN','reason':str(error)}
|
|
except ValueError as error: results[name]={'status':'FAIL','reason':str(error)}
|
|
for name,path in [('performance',RAW/'performance/result.json'),('windows-pdf',RAW/'native-pdf.json'),('linux-pdf',ROOT/'.artifacts/p9-sec-02/final-linux-02/linux-pdf.json')]:
|
|
try:
|
|
require_fresh(path)
|
|
data=prior.load(path)
|
|
if name.endswith('pdf'):
|
|
if hashlib.sha256((ROOT/'scripts/verify-cv-fidelity.py').read_bytes()).hexdigest()!='64f82e622cc11483c33307d987e969f6cf27f82ab6bdbd72c2aeb1bfbceade87': raise ValueError('Original PDF oracle changed.')
|
|
if data.get('inputDrift') or any(hashlib.sha256(Path(file).read_bytes()).hexdigest()!=digest for file,digest in data['inputs'].items()): raise ValueError('PDF inputs changed after qualification.')
|
|
if len(data['lanes'])!=2 or any(lane['status']!='PASS' or lane['exitCode']!=0 or hashlib.sha256(Path(lane['report']).read_bytes()).hexdigest()!=lane['reportSha256'] for lane in data['lanes']): raise ValueError('PDF lane/report failure or drift.')
|
|
results[name]={'status':data['status'],'record':data,'evidence':prior.identity(path)}
|
|
except FileNotFoundError as error: results[name]={'status':'NOT_RUN','reason':str(error)}
|
|
except ValueError as error: results[name]={'status':'FAIL','reason':str(error)}
|
|
try:
|
|
path=RAW/'security/commands.json';commands=prior.load(path)
|
|
passed,vulnerabilities=prior.security_results(commands,prior.load(RAW/'security/frontend-audit.json'),prior.load(RAW/'security/nuget-advisories.log'))
|
|
results['security-helpers']={'status':'PASS' if passed else 'FAIL','commands':commands,'evidence':prior.identity(path),'nugetVulnerabilities':vulnerabilities,'scope':'Twelve bounded helpers/audits; not Python advisory closure.'}
|
|
except FileNotFoundError as error: results['security-helpers']={'status':'NOT_RUN','reason':str(error)}
|
|
except ValueError as error: results['security-helpers']={'status':'FAIL','reason':str(error)}
|
|
for name, path in [('model-runtime', ROOT/'.artifacts/p9-sec-02/model-regression/result.json'),
|
|
('trained-model', ROOT/'.artifacts/p9-sec-02/full-model/result.json')]:
|
|
try:
|
|
data=prior.load(path)
|
|
if data['status']!='PASS': raise ValueError('Actual model qualification did not pass.')
|
|
if name=='model-runtime':
|
|
require_unchanged({str(ROOT/n):h for n,h in data['inputs'].items()})
|
|
if set(data['results'])!={'sidecar','inventory','safetensors','weights-only','cache','parsers'}: raise ValueError('Model lane membership differs.')
|
|
for lane in data['results'].values():
|
|
process=lane['process']
|
|
if lane['status']!='PASS' or process['exitCode']!=0 or process.get('containerAbsent') is not True or process.get('cleanupStatus')!='ABSENT': raise ValueError('Model lane process/positive cleanup evidence missing.')
|
|
ocr=prior.load(HERE/'model/OCR_ARTIFACT.json')
|
|
native=data['results']['inventory']['record']['englishOcrData']
|
|
if native['sha256']!=ocr['sha256'] or native['bytes']!=ocr['bytes'] or prior.identity(ROOT/ocr['licenseEvidence']['path'])['sha256']!=ocr['licenseEvidence']['sha256']:
|
|
raise ValueError('Native OCR identity/licence evidence differs.')
|
|
capsule=prior.load(HERE/'model/fixture-capsule.json')
|
|
if prior.identity(ROOT/capsule['path'])['sha256']!=capsule['sha256']: raise ValueError('Local small-fixture replay capsule changed.')
|
|
sidecar=ROOT/'.artifacts/p9-sec-02/qualified/model-regression-sidecar/sidecar.xml'
|
|
cases=prior.sidecar_results(ET.parse(sidecar).getroot(),59)
|
|
reference=ROOT/'.artifacts/p9-remed-02/final/sidecar.xml'
|
|
accepted=prior.load(HERE.parent/'p9-remed-02/validation.json')['results']['sidecar']['report']['sha256']
|
|
if prior.identity(reference)['sha256']!=accepted: raise ValueError('Historical sidecar membership reference drift.')
|
|
previous=prior.sidecar_results(ET.parse(reference).getroot(),58)
|
|
actual={(c['classname'],c['name']) for c in cases}; old={(c['classname'],c['name']) for c in previous}
|
|
added=actual-old
|
|
if not old.issubset(actual) or len(added)!=1 or next(iter(added))[1]!='test_model_load_pins_both_artifacts_and_denies_remote_code_and_unsafe_weights': raise ValueError('Sidecar membership differs from58 plus declared pin test.')
|
|
results['sidecar']={'status':'PASS','passed':59,'cases':cases,'report':prior.identity(sidecar),'membershipReference':prior.identity(reference)}
|
|
else:
|
|
if (data['exitCode']!=0 or data.get('containerAbsent') is not True
|
|
or data.get('downloadBytes')!=1222317369
|
|
or data.get('downloadSha256')!='3bac65d18c99463302d12ca75c2220ea714f9c81ce235f205fa818efe71df6ea'
|
|
or data['result'].get('status')!='PASS' or data['result'].get('modelClass')!='BartForConditionalGeneration'
|
|
or data['result'].get('device')!='cpu'
|
|
or data['result'].get('repositoryRevision')!='a4f8f3ea906ed274767e9906dbaede7531d660ff'
|
|
or data['result'].get('manifestSha256')!=prior.identity(HERE/'full-model/ARTIFACTS.json')['sha256']): raise ValueError('Full trained-checkpoint identity/process outcome differs.')
|
|
results[name]={'status':'PASS','record':data,'evidence':prior.identity(path)}
|
|
except FileNotFoundError as error: results[name]={'status':'NOT_RUN','reason':str(error)}
|
|
except (ValueError,KeyError) as error: results[name]={'status':'FAIL','reason':str(error)}
|
|
try:
|
|
scan=ROOT/'.artifacts/p9-sec-02/final-advisories/summary.json'
|
|
current=prior.load(scan)
|
|
expected_gap=[{'package':'torch','version':'2.13.0+cpu','provider':'pypi','httpStatus':404,'error':'HTTPError'}]
|
|
if current.get('packageCount')!=63 or current.get('sourceQueryCount')!=126 or current.get('errors')!=expected_gap:
|
|
raise ValueError('Installed scan membership/query coverage differs; do not convert provider errors into clean results.')
|
|
installed=results['model-runtime']['record']['results']['inventory']['record']['installedDistributions']
|
|
audited={row['name']:row['version'] for row in current['packages']}
|
|
if audited!=installed: raise ValueError('Scanned package graph differs from actual qualified installed graph.')
|
|
matrix=prior.load(HERE/'DISPOSITIONS.json')
|
|
from collections import Counter
|
|
dispositions=Counter(row['disposition'] for row in matrix['originalGroups'])
|
|
if len(matrix['originalGroups'])!=109 or len(matrix['additionalGroups'])!=1 or dispositions!={'Fixed and verified':99,'Not affected, with evidence':10}:
|
|
raise ValueError('Original109/new1 advisory accounting differs.')
|
|
results['python-advisories']={'status':'INCOMPLETE' if current['advisoryRecords']==0 else 'FINDINGS','record':current,
|
|
'scopedAssessmentEvidenceComplete':current['advisoryRecords']==0,
|
|
'scan':prior.identity(scan),'dispositions':prior.identity(HERE/'DISPOSITIONS.json'),
|
|
'coverage':'Exact installed OSV/PyPI queries; known vendor PyPI404 for CPU-local Torch retained. Official CPU wheel/vendor/source evidence supplements that unavailable query; no complete two-provider or deployed/image-wide security claim.'}
|
|
except FileNotFoundError as error: results['python-advisories']={'status':'NOT_RUN','reason':str(error)}
|
|
except (ValueError,KeyError) as error: results['python-advisories']={'status':'FAIL','reason':str(error)}
|
|
try:
|
|
path=ROOT/'.artifacts/p9-sec-02/final-guards/result.json'; guards=prior.load(path)
|
|
expected={'model-integrity-cleanup':10,'artifact-preparation':3,'fresh-evidence':3,'workflow-boundary':4}
|
|
if (guards['status']!='PASS' or guards['passed']!=20 or len(guards['results'])!=4
|
|
or {row['name']:row['expected'] for row in guards['results']}!=expected
|
|
or any(row['status']!='PASS' or row['exitCode']!=0 or prior.identity(path.parent/(row['name']+'.log'))['sha256']!=row['logSha256'] for row in guards['results'])):
|
|
raise ValueError('Supplementary harness guard membership/outcome/log identity differs.')
|
|
results['harness-guards']={'status':'PASS','passed':20,'record':guards,'evidence':prior.identity(path)}
|
|
except FileNotFoundError as error: results['harness-guards']={'status':'NOT_RUN','reason':str(error)}
|
|
except (ValueError,KeyError) as error: results['harness-guards']={'status':'FAIL','reason':str(error)}
|
|
try:
|
|
cleanup=prior.load(HERE/'cleanup.json')
|
|
if (cleanup['status']!='PASS' or any(cleanup.get(n) is not True for n in ['generatedSchemasAbsentBeforeRemoval','containerAbsent','labelledModelContainersAbsent','qualificationApiContainersAbsent'])
|
|
or cleanup['developmentContainer']['stateAndMountsUnchanged'] is not True or cleanup['developmentVolume']['metadataUnchanged'] is not True):
|
|
raise ValueError('Disposable cleanup/development preservation evidence incomplete.')
|
|
for name,record in cleanup['serverLogs'].items():
|
|
if prior.identity(ROOT/'.artifacts/p9-sec-02/database-cleanup'/name)['sha256']!=record['sha256']: raise ValueError('Retained database log stream changed.')
|
|
results['disposable-cleanup']={'status':'PASS','record':cleanup,'evidence':prior.identity(HERE/'cleanup.json')}
|
|
except FileNotFoundError as error: results['disposable-cleanup']={'status':'NOT_RUN','reason':str(error)}
|
|
except (ValueError,KeyError) as error: results['disposable-cleanup']={'status':'FAIL','reason':str(error)}
|
|
results['hosted-mariadb']={'status':'BLOCKED','executedMariaDbCases':0,'runnerIsolation':'UNVERIFIED','evidence':prior.identity(HERE/'ci-assessment.json'),'queuedEvidence':prior.identity(HERE/'ci-postpush.json'),'finalMetadata':prior.identity(HERE/'ci-final-refresh.json')}
|
|
manifest=prior.load(RAW/'source-freeze.json')
|
|
helpers=prior.load(ROOT/'.artifacts/p9-sec-02/final-helper-inputs.json')
|
|
drift=[n for n,h in manifest['inputs'].items() if not (ROOT/n).is_file() or hashlib.sha256((ROOT/n).read_bytes()).hexdigest()!=h]
|
|
if amendment and results['ordinary-3']['status']=='PASS':
|
|
name=amendment['file']
|
|
if manifest['inputs'].get(name)!=amendment['before']: raise ValueError('Fixture successor does not derive from frozen predecessor.')
|
|
drift=[n for n in drift if n!=name]
|
|
helper_drift=[n for n,h in helpers['inputs'].items() if not (ROOT/n).is_file() or hashlib.sha256((ROOT/n).read_bytes()).hexdigest()!=h]
|
|
local={k:v for k,v in results.items() if k not in ['hosted-mariadb','python-advisories']}
|
|
record={'schemaVersion':1,'observedUtc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
'baseline':'f17dba2ec06ea8f53c2785eaaebd25abc515ebf8','branch':'fix/p9-model-cache-and-ci-qualification',
|
|
'applicationCheckpoint':manifest['applicationCheckpoint'],'sourceIdentity':prior.identity(RAW/'source-freeze.json'),
|
|
'frozenInputCount':len(manifest['inputs']),'runtimeDrift':drift,'helperIdentity':prior.identity(ROOT/'.artifacts/p9-sec-02/final-helper-inputs.json'),
|
|
'frozenHelperCount':len(helpers['inputs']),'helperDrift':helper_drift,
|
|
'declaredFixtureSuccessor':amendment,
|
|
'retainedReportingDryRun':{'evidence':prior.identity(ROOT/'.artifacts/p9-sec-02/collector-initial-dry-run.json'),
|
|
'scope':'Unpublished initial collector dry run exposed an inherited historical Linux PDF path. Corrected before acceptance; fresh-root negative guards prevent historical results becoming fresh passes.'},
|
|
'countPolicy':'Separate lanes. Superseded/focused runs are retained and never added to successor totals.',
|
|
'localFunctionalComplete':not drift and not helper_drift and all(v['status']=='PASS' for v in local.values()),
|
|
'automationComplete':False,'results':results}
|
|
(HERE/'validation.json').write_text(json.dumps(record,indent=2)+'\n',encoding='utf-8')
|
|
print(json.dumps({n:v['status'] for n,v in results.items()}))
|
|
if drift or helper_drift: raise SystemExit('Frozen input drift: '+str(drift+helper_drift))
|
|
if '--collect-only' not in sys.argv: prior.require_complete(results)
|
|
|
|
if __name__=='__main__':main()
|