cesnimda
378 lines
21 KiB
Python
378 lines
21 KiB
Python
"""Offline current-source image preparation and disposable Linux PDF qualification.
|
|
|
|
Build and run are explicit, separate commands. No dependency download or production service.
|
|
"""
|
|
import argparse
|
|
import datetime
|
|
import hashlib
|
|
import http.cookiejar
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import re
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.request
|
|
import uuid
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
BASE = 'sha256:0d1cb44ba8f71d54de77459915a7052b894191cfeb75aa32f0c1e7ffdc21fda6'
|
|
PIN = 'mariadb:12.2.2@sha256:42a27baaba3e8f3f0159d16381d5a7156661aa91dcc65f663cabb6a6f9503c00'
|
|
APP = '/app/cv-ux-current'
|
|
CONFIG = '../docs/cv-redesign/linux/playwright.config.ts'
|
|
PASSWORD = 'jt-integration-only'
|
|
AUTH_PASSWORD = 'E2ePassword123!'
|
|
JWT = 'e2e-only-key-at-least-thirty-two-characters-long'
|
|
|
|
|
|
def utc():
|
|
return datetime.datetime.now(datetime.timezone.utc).isoformat()
|
|
|
|
|
|
def scrub(value):
|
|
text = str(value)
|
|
for secret in (PASSWORD, AUTH_PASSWORD, JWT):
|
|
text = text.replace(secret, '[synthetic-test-secret]')
|
|
return text
|
|
|
|
|
|
def digest(value):
|
|
return hashlib.sha256(value).hexdigest()
|
|
|
|
|
|
def identity(files):
|
|
return digest(json.dumps(files, sort_keys=True, separators=(',', ':')).encode())
|
|
|
|
|
|
def write(path, value):
|
|
path.write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8')
|
|
|
|
|
|
def fresh(path):
|
|
path = Path(path).resolve()
|
|
allowed = (ROOT / '.artifacts/cv-ux-01').resolve()
|
|
if not path.is_relative_to(allowed) or path == allowed or path.exists():
|
|
raise ValueError('Select a nonexistent child of .artifacts/cv-ux-01; previous attempts are immutable.')
|
|
path.mkdir(parents=True)
|
|
return path
|
|
|
|
|
|
class Commands:
|
|
def __init__(self, out):
|
|
self.out = out
|
|
self.records = []
|
|
|
|
def run(self, args, *, env=None, cwd=ROOT, check=True, timeout=600):
|
|
item = {'startedUtc': utc(), 'argv': [scrub(a) for a in args], 'cwd': str(cwd)}
|
|
self.records.append(item)
|
|
try:
|
|
result = subprocess.run(['rtk', 'proxy', *map(str, args)], cwd=cwd, env=env,
|
|
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout)
|
|
item['exitCode'] = result.returncode
|
|
output = result.stdout + result.stderr
|
|
log = self.out / f'command-{len(self.records):03}.log'
|
|
log.write_text(scrub(output), encoding='utf-8')
|
|
item['log'] = str(log)
|
|
item['logSha256'] = digest(log.read_bytes())
|
|
if check and result.returncode:
|
|
raise RuntimeError(f'Command {len(self.records)} failed with exit {result.returncode}; inspect {log.name}')
|
|
return result.stdout.strip()
|
|
finally:
|
|
item['finishedUtc'] = utc()
|
|
write(self.out / 'commands.json', self.records)
|
|
|
|
|
|
def sources(commands):
|
|
paths = commands.run(['git', '-c', 'core.quotePath=false', 'ls-files', '--cached', '--others', '--exclude-standard']).splitlines()
|
|
prefixes = ('JobTrackerApi/', 'job-tracker-ui/src/', 'job-tracker-ui/e2e/')
|
|
exact = {'global.json', '.dockerignore', 'docker-compose.database.yml', 'NuGet.Config', 'nuget.config', 'job-tracker-ui/package.json', 'job-tracker-ui/package-lock.json',
|
|
'scripts/verify-cv-fidelity.py', 'docs/redesign/phase-9/p9-requal-01/pdf-proof.py'}
|
|
selected = [name for name in paths if name.startswith(prefixes) or name in exact
|
|
or name.startswith(('Directory.', 'job-tracker-ui/playwright', 'job-tracker-ui/scripts/'))
|
|
or name.startswith('docs/cv-redesign/') and name.endswith(('.py', '.ts', '.json'))]
|
|
return {name: digest((ROOT / name).read_bytes()) for name in sorted(set(selected)) if (ROOT / name).is_file()}
|
|
|
|
|
|
def tree(path):
|
|
return {file.relative_to(path).as_posix(): digest(file.read_bytes()) for file in sorted(path.rglob('*')) if file.is_file()}
|
|
|
|
|
|
def image_id(value):
|
|
if not re.fullmatch(r'sha256:[a-f0-9]{64}', value):
|
|
raise ValueError('An immutable local image ID is required.')
|
|
return value
|
|
|
|
|
|
def schema_name(value):
|
|
if not re.fullmatch(r'jt_it_[a-f0-9]{32}', value):
|
|
raise ValueError('Only a generated disposable schema is allowed.')
|
|
return value
|
|
|
|
|
|
def guard_db(data, name, project, port):
|
|
if not re.fullmatch(r'jobtracker-it-[a-f0-9]{32}', project) or name != project + '-mariadb-test-1':
|
|
raise ValueError('Unexpected generated DB owner.')
|
|
labels = data.get('Config', {}).get('Labels', {})
|
|
host = data.get('HostConfig', {})
|
|
if (data.get('Name') != '/' + name or data['Config']['Image'] != PIN
|
|
or labels.get('com.docker.compose.project') != project
|
|
or labels.get('com.docker.compose.service') != 'mariadb-test'
|
|
or not data['State']['Running'] or data['State'].get('Health', {}).get('Status') != 'healthy'
|
|
or host.get('Privileged') or host.get('NetworkMode') == 'host'
|
|
or '/var/lib/mysql' not in host.get('Tmpfs', {})
|
|
or host.get('PortBindings', {}).get('3306/tcp') != [{'HostIp': '127.0.0.1', 'HostPort': str(port)}]
|
|
or any(mount.get('Destination') == '/var/lib/mysql' and mount.get('Type') != 'tmpfs' for mount in data.get('Mounts', []))):
|
|
raise ValueError('DB must be the exact healthy pinned loopback tmpfs test service.')
|
|
|
|
|
|
def verify_build(record, expected_image, current_sources):
|
|
if (record.get('exitCode') != 0 or record.get('baseImageId') != BASE
|
|
or record.get('imageId') != image_id(expected_image)
|
|
or record.get('applicationPath') != APP
|
|
or record.get('sourceSha256') != identity(current_sources)
|
|
or record.get('sources') != current_sources):
|
|
raise ValueError('Current source/build/image identity differs; rebuild in a fresh root.')
|
|
|
|
|
|
def build(args):
|
|
out = fresh(args.output)
|
|
commands = Commands(out)
|
|
record = {'startedUtc': utc(), 'exitCode': 1, 'baseImageId': BASE, 'applicationPath': APP}
|
|
try:
|
|
record['sources'] = sources(commands)
|
|
record['sourceSha256'] = identity(record['sources'])
|
|
record['commit'] = commands.run(['git', 'rev-parse', 'HEAD'])
|
|
record['status'] = commands.run(['git', 'status', '--short'])
|
|
record['dotnetInfo'] = commands.run([args.dotnet, '--info'])
|
|
assert commands.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', BASE]) == BASE
|
|
record['baseMetadata'] = json.loads(commands.run(['docker', 'image', 'inspect', '--format', '{{json .Config.Labels}}', BASE]))
|
|
native_tag = 'jobtracker-cv-ux-native:' + uuid.uuid4().hex
|
|
commands.run(['docker', 'tag', BASE, native_tag])
|
|
assert commands.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', native_tag]) == BASE
|
|
record['baseImageTag'] = native_tag
|
|
publish = out / 'publish'
|
|
env = {**os.environ, 'DOTNET_ROOT': str(Path(args.dotnet).parent)}
|
|
record['publishEnvironment'] = {'DOTNET_ROOT': env['DOTNET_ROOT'], 'NuGetAudit': env.get('NuGetAudit')}
|
|
commands.run([args.dotnet, 'publish', 'JobTrackerApi/JobTrackerApi.csproj', '--configuration', 'Release',
|
|
'--no-restore', '--output', str(publish), '/p:UseAppHost=false', '--self-contained', 'false'], env=env)
|
|
record['publish'] = tree(publish)
|
|
record['publishSha256'] = identity(record['publish'])
|
|
if 'JobTrackerApi.dll' not in record['publish'] or 'JobTrackerApi.deps.json' not in record['publish']:
|
|
raise ValueError('Portable current API output is incomplete.')
|
|
dockerfile = out / 'Dockerfile'
|
|
dockerfile.write_text(f'FROM {native_tag}\nWORKDIR {APP}\nCOPY publish/ ./\n'
|
|
+ f'LABEL io.jobjakt.cv-ux.source-sha256="{record["sourceSha256"]}" '
|
|
+ f'io.jobjakt.cv-ux.publish-sha256="{record["publishSha256"]}"\n'
|
|
+ 'ENTRYPOINT ["dotnet","JobTrackerApi.dll"]\n', encoding='utf-8')
|
|
record['dockerfileSha256'] = digest(dockerfile.read_bytes())
|
|
tag = 'jobtracker-cv-ux-api:' + uuid.uuid4().hex
|
|
commands.run(['docker', 'build', '--network', 'none', '--pull=false', '--file', str(dockerfile), '--tag', tag, str(out)])
|
|
record['imageTag'] = tag
|
|
record['imageId'] = image_id(commands.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', tag]))
|
|
if record['sources'] != sources(commands) or record['publish'] != tree(publish):
|
|
raise ValueError('Source or publish bytes changed during build.')
|
|
record['exitCode'] = 0
|
|
except Exception as error:
|
|
record['error'] = scrub(error)
|
|
finally:
|
|
record['finishedUtc'] = utc()
|
|
write(out / 'build.json', record)
|
|
print(json.dumps({'exitCode': record['exitCode'], 'record': str(out / 'build.json')}))
|
|
return record['exitCode']
|
|
|
|
|
|
def free_port(port=0):
|
|
with socket.socket() as probe:
|
|
probe.bind(('127.0.0.1', port))
|
|
return probe.getsockname()[1]
|
|
|
|
|
|
def auth_identity(version, commit):
|
|
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
|
|
request = urllib.request.Request('http://localhost:5302/api/auth/login',
|
|
data=json.dumps({'email': 'e2e@example.test', 'password': AUTH_PASSWORD}).encode(),
|
|
headers={'Content-Type': 'application/json', 'Origin': 'http://localhost:3300'})
|
|
with opener.open(request, timeout=10) as response:
|
|
response.read()
|
|
with opener.open('http://localhost:5302/api/auth/me', timeout=10) as response:
|
|
me = json.load(response)
|
|
if me.get('appVersion') != version or me.get('appCommitSha') != commit:
|
|
raise ValueError('Authenticated API build metadata mismatch.')
|
|
return {'appVersion': me['appVersion'], 'appCommitSha': me['appCommitSha']}
|
|
|
|
|
|
def run(args):
|
|
out = fresh(args.output)
|
|
commands = Commands(out)
|
|
record = {'startedUtc': utc(), 'exitCode': 1, 'cleanup': {}}
|
|
suffix = uuid.uuid4().hex
|
|
project = 'jobtracker-it-' + suffix
|
|
db = project + '-mariadb-test-1'
|
|
network = 'jobtracker-cv-ux-' + suffix
|
|
api = network + '-api'
|
|
schema = schema_name('jt_it_' + uuid.uuid4().hex)
|
|
compose = ['docker', 'compose', '--project-name', project, '--project-directory', str(ROOT),
|
|
'-f', str(ROOT / 'docker-compose.database.yml'), '--profile', 'integration']
|
|
db_attempted = created = attached = network_created = api_created = False
|
|
|
|
def sql(query):
|
|
return commands.run(['docker', 'exec', '-e', 'MYSQL_PWD=' + PASSWORD, db,
|
|
'mariadb', '-uroot', '--batch', '--skip-column-names', '-e', query])
|
|
|
|
def clean(key, action):
|
|
try:
|
|
action()
|
|
record['cleanup'][key] = True
|
|
except Exception as error:
|
|
record['cleanup'][key + 'Error'] = scrub(error)
|
|
record['exitCode'] = 1
|
|
|
|
try:
|
|
build_record = json.loads(Path(args.build_record).read_text(encoding='utf-8'))
|
|
current_sources = sources(commands)
|
|
verify_build(build_record, args.image, current_sources)
|
|
publish = Path(args.build_record).resolve().parent / 'publish'
|
|
if tree(publish) != build_record['publish']:
|
|
raise ValueError('Published API output drifted after build.')
|
|
static = ROOT / 'job-tracker-ui/out'
|
|
record['frontend'] = tree(static)
|
|
if not record['frontend']:
|
|
raise ValueError('Prebuilt frontend output is required.')
|
|
record.update({'buildRecord': str(Path(args.build_record).resolve()), 'buildRecordSha256': digest(Path(args.build_record).read_bytes()),
|
|
'sourceSha256': build_record['sourceSha256'], 'imageId': args.image, 'commit': build_record['commit'],
|
|
'schema': schema, 'dbContainer': db, 'dbProject': project, 'apiContainer': api, 'network': network})
|
|
image = json.loads(commands.run(['docker', 'image', 'inspect', args.image]))[0]
|
|
if (image['Id'] != args.image or image['Config']['WorkingDir'] != APP
|
|
or image['Config'].get('Labels', {}).get('io.jobjakt.cv-ux.source-sha256') != record['sourceSha256']
|
|
or image['Config'].get('Labels', {}).get('io.jobjakt.cv-ux.publish-sha256') != build_record['publishSha256']):
|
|
raise ValueError('Image source/publish labels or current app directory mismatch.')
|
|
free_port(5302)
|
|
free_port(3300)
|
|
record['chromium'] = commands.run(['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/usr/bin/chromium', args.image, '--version'])
|
|
record['runtime'] = commands.run(['docker', 'run', '--rm', '--network', 'none', '--entrypoint', 'dotnet', args.image, '--info'])
|
|
record['apiDllSha256'] = commands.run(['docker', 'run', '--rm', '--network', 'none', '--entrypoint', 'sha256sum', args.image, APP + '/JobTrackerApi.dll']).split()[0]
|
|
if record['apiDllSha256'] != build_record['publish']['JobTrackerApi.dll']:
|
|
raise ValueError('Image API DLL differs from current publish bytes.')
|
|
port = free_port()
|
|
record['dbPort'] = port
|
|
environment = {**os.environ, 'MARIADB_TEST_PORT': str(port)}
|
|
assert commands.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', PIN])
|
|
db_attempted = True
|
|
commands.run(compose + ['up', '--pull', 'never', '-d', '--wait', '--wait-timeout', '120', 'mariadb-test'], env=environment)
|
|
guard_db(json.loads(commands.run(['docker', 'inspect', db]))[0], db, project, port)
|
|
record['serverVersion'] = sql('SELECT VERSION()')
|
|
if not record['serverVersion'].startswith('12.2.2-MariaDB'):
|
|
raise ValueError('Exact MariaDB 12.2.2 is required.')
|
|
assert sql(f"SELECT COUNT(*) FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='{schema}'") == '0'
|
|
sql(f'CREATE DATABASE `{schema}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci')
|
|
created = True
|
|
commands.run(['docker', 'network', 'create', '--internal', '--label', 'task=cv-ux-01', network])
|
|
network_created = True
|
|
commands.run(['docker', 'network', 'connect', '--alias', 'mariadb', network, db])
|
|
attached = True
|
|
version = 'cv-ux-linux-' + record['sourceSha256'][:12]
|
|
variables = {'ASPNETCORE_ENVIRONMENT': 'Development', 'Database__Provider': 'mariadb', 'Database__ServerVersion': '12.2.2',
|
|
'ConnectionStrings__JobTracker': f'Server=mariadb;Database={schema};User ID=root;Password={PASSWORD};SslMode=None',
|
|
'Auth__Require': 'true', 'Auth__JwtKey': JWT, 'Auth__AdminEmail': 'e2e@example.test', 'Auth__AdminPassword': AUTH_PASSWORD,
|
|
'Auth__AllowRegistration': 'false', 'RateLimits__AuthLoginPermitLimit': '1000', 'Cors__Origins__0': 'http://localhost:3300',
|
|
'Workers__AiOperationsEnabled': 'false', 'Workers__SavedJobSearchAlertsEnabled': 'false', 'HttpsRedirection__Enabled': 'false',
|
|
'App__Version': version, 'App__CommitSha': record['commit'], 'Data__Root': '/data', 'Logging__LogLevel__Default': 'Warning'}
|
|
create = ['docker', 'create', '--name', api, '--label', 'task=cv-ux-01', '--network', network,
|
|
'-p', '127.0.0.1:5302:8080', '--tmpfs', '/data:rw,nosuid,size=512m']
|
|
for key, value in variables.items():
|
|
create += ['-e', key + '=' + value]
|
|
api_created = True
|
|
commands.run(create + [args.image])
|
|
assert commands.run(['docker', 'inspect', '--format', '{{.Image}}', api]) == args.image
|
|
commands.run(['docker', 'start', api])
|
|
deadline = time.monotonic() + 120
|
|
while True:
|
|
try:
|
|
with urllib.request.urlopen('http://localhost:5302/health', timeout=3) as response:
|
|
health = json.load(response)
|
|
if health.get('version') != version:
|
|
raise ValueError('Linux API health version mismatch.')
|
|
with urllib.request.urlopen('http://localhost:5302/ready', timeout=3) as response:
|
|
assert json.load(response)['status'] == 'ready'
|
|
record['health'] = health
|
|
record['apiBuildMetadata'] = auth_identity(version, record['commit'])
|
|
break
|
|
except Exception:
|
|
if time.monotonic() >= deadline:
|
|
raise
|
|
time.sleep(1)
|
|
migration_sql = f'SELECT MigrationId FROM `{schema}`.`__EFMigrationsHistory` ORDER BY MigrationId'
|
|
record['migrations'] = sql(migration_sql).splitlines()
|
|
if '20261025090000_AddCoverLetterDocumentDesign' not in record['migrations']:
|
|
raise ValueError('Current letter design migration was not applied.')
|
|
env = {**os.environ, 'CV_UX_LINUX_READY': version, 'CV_UX_LINUX_SOURCE': record['sourceSha256'],
|
|
'PLAYWRIGHT_STATIC_EXPORT': 'true', 'PLAYWRIGHT_PREBUILT_EXPORT': 'true', 'PLAYWRIGHT_REUSE_SERVERS': 'false',
|
|
'NEXT_PUBLIC_API_BASE_URL': 'http://localhost:5302/api', 'CV_FIDELITY_OUTPUT': str(out / 'fidelity'),
|
|
'CV_SETTINGS_FIDELITY_OUTPUT': str(out / 'settings-fidelity'), 'LETTER_FIDELITY_OUTPUT': str(out / 'letter-fidelity'),
|
|
'PLAYWRIGHT_JSON_OUTPUT_NAME': str(out / 'linux-renderer.json')}
|
|
commands.run(['node', 'node_modules/@playwright/test/cli.js', 'test', '--config=' + CONFIG, '--retries=0',
|
|
'--reporter=list,json', '--output=' + str(out / 'test-results')], cwd=ROOT / 'job-tracker-ui', env=env, timeout=1800)
|
|
native = json.loads((out / 'linux-renderer.json').read_text(encoding='utf-8'))
|
|
stats = native.get('stats', {})
|
|
if stats.get('expected') != 6 or any(stats.get(key) != 0 for key in ('unexpected', 'flaky', 'skipped')) or native.get('errors'):
|
|
raise ValueError('Expected six passing browser cases with no skips, retries or errors.')
|
|
record['browserStats'] = stats
|
|
if sql(migration_sql).splitlines() != record['migrations'] or sources(commands) != current_sources or tree(static) != record['frontend']:
|
|
raise ValueError('Source/runtime/migration drift during rendering.')
|
|
for folder in ('fidelity', 'letter-fidelity'):
|
|
if not (out / folder / 'measurements.json').is_file():
|
|
raise ValueError('Native corpus measurements missing: ' + folder)
|
|
cv = json.loads((out / 'fidelity/measurements.json').read_text(encoding='utf-8'))
|
|
letter = json.loads((out / 'letter-fidelity/measurements.json').read_text(encoding='utf-8'))
|
|
settings_files = list((out / 'settings-fidelity').glob('*-measurements.json'))
|
|
if len(cv.get('fixtures', [])) != 16 or len(settings_files) != 4 or sum(len(json.loads(file.read_text(encoding='utf-8')).get('fixtures', [])) for file in settings_files) != 8:
|
|
raise ValueError('The original CV16 + settings8 document membership is incomplete.')
|
|
if {fixture.get('fixture') for fixture in letter.get('fixtures', [])} != {'light-en-a4', 'dark-nb-a4', 'light-nb-letter', 'dark-en-letter'}:
|
|
raise ValueError('The independent letter4 document membership is incomplete.')
|
|
record['exitCode'] = 0
|
|
except Exception as error:
|
|
record['error'] = scrub(error)
|
|
finally:
|
|
if api_created:
|
|
clean('apiLogsCaptured', lambda: commands.run(['docker', 'logs', api]))
|
|
clean('apiRemoved', lambda: commands.run(['docker', 'rm', '--force', api]))
|
|
if attached:
|
|
clean('databaseDisconnected', lambda: commands.run(['docker', 'network', 'disconnect', network, db]))
|
|
if network_created:
|
|
clean('networkRemoved', lambda: commands.run(['docker', 'network', 'rm', network]))
|
|
if created and (not api_created or record['cleanup'].get('apiRemoved')):
|
|
def remove_schema():
|
|
sql(f'DROP DATABASE `{schema_name(schema)}`')
|
|
assert sql(f"SELECT COUNT(*) FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='{schema}'") == '0'
|
|
clean('schemaRemoved', remove_schema)
|
|
elif created:
|
|
record['cleanup']['schemaRemovedError'] = 'API removal unconfirmed; schema retained.'
|
|
record['exitCode'] = 1
|
|
if db_attempted:
|
|
clean('dbLogsCaptured', lambda: commands.run(compose + ['logs', '--no-color', 'mariadb-test']))
|
|
clean('databaseRemoved', lambda: commands.run(compose + ['rm', '--stop', '--force', 'mariadb-test']))
|
|
record['finishedUtc'] = utc()
|
|
write(out / 'process.json', record)
|
|
print(json.dumps({'exitCode': record['exitCode'], 'record': str(out / 'process.json')}))
|
|
return record['exitCode']
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
modes = parser.add_subparsers(dest='mode', required=True)
|
|
image = modes.add_parser('build')
|
|
image.add_argument('--output', required=True)
|
|
image.add_argument('--dotnet', default='C:/Users/Connor/.dotnet/dotnet.exe')
|
|
renderer = modes.add_parser('run')
|
|
renderer.add_argument('--output', required=True)
|
|
renderer.add_argument('--build-record', required=True)
|
|
renderer.add_argument('--image', required=True)
|
|
args = parser.parse_args()
|
|
return build(args) if args.mode == 'build' else run(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|