Files
jobtrackingapp/docs/cv-redesign/linux-run.py
T

382 lines
21 KiB
Python

"""Offline current-source image preparation and disposable Linux PDF qualification.
Build and run are explicit, separate commands. No dependency download or production service.
"""
import argparse
import datetime
import hashlib
import http.cookiejar
import json
import os
from pathlib import Path
import re
import socket
import subprocess
import sys
import time
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[2]
BASE = 'sha256:0d1cb44ba8f71d54de77459915a7052b894191cfeb75aa32f0c1e7ffdc21fda6'
PIN = 'mariadb:12.2.2@sha256:42a27baaba3e8f3f0159d16381d5a7156661aa91dcc65f663cabb6a6f9503c00'
APP = '/app/cv-ux-current'
CONFIG = '../docs/cv-redesign/linux/playwright.config.ts'
PASSWORD = 'jt-integration-only'
AUTH_PASSWORD = 'E2ePassword123!'
JWT = 'e2e-only-key-at-least-thirty-two-characters-long'
def utc():
return datetime.datetime.now(datetime.timezone.utc).isoformat()
def scrub(value):
text = str(value)
for secret in (PASSWORD, AUTH_PASSWORD, JWT):
text = text.replace(secret, '[synthetic-test-secret]')
return text
def digest(value):
return hashlib.sha256(value).hexdigest()
def identity(files):
return digest(json.dumps(files, sort_keys=True, separators=(',', ':')).encode())
def write(path, value):
path.write_text(json.dumps(value, indent=2) + '\n', encoding='utf-8')
def fresh(path):
path = Path(path).resolve()
allowed = (ROOT / '.artifacts/cv-ux-01').resolve()
if not path.is_relative_to(allowed) or path == allowed or path.exists():
raise ValueError('Select a nonexistent child of .artifacts/cv-ux-01; previous attempts are immutable.')
path.mkdir(parents=True)
return path
class Commands:
def __init__(self, out):
self.out = out
self.records = []
def run(self, args, *, env=None, cwd=ROOT, check=True, timeout=600):
item = {'startedUtc': utc(), 'argv': [scrub(a) for a in args], 'cwd': str(cwd)}
self.records.append(item)
try:
result = subprocess.run(['rtk', 'proxy', *map(str, args)], cwd=cwd, env=env,
capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout)
item['exitCode'] = result.returncode
output = result.stdout + result.stderr
log = self.out / f'command-{len(self.records):03}.log'
log.write_text(scrub(output), encoding='utf-8')
item['log'] = str(log)
item['logSha256'] = digest(log.read_bytes())
if check and result.returncode:
raise RuntimeError(f'Command {len(self.records)} failed with exit {result.returncode}; inspect {log.name}')
return result.stdout.strip()
finally:
item['finishedUtc'] = utc()
write(self.out / 'commands.json', self.records)
def sources(commands):
paths = commands.run(['git', '-c', 'core.quotePath=false', 'ls-files', '--cached', '--others', '--exclude-standard']).splitlines()
prefixes = ('JobTrackerApi/', 'job-tracker-ui/src/', 'job-tracker-ui/e2e/')
exact = {'global.json', '.dockerignore', 'docker-compose.database.yml', 'NuGet.Config', 'nuget.config', 'job-tracker-ui/package.json', 'job-tracker-ui/package-lock.json',
'scripts/verify-cv-fidelity.py', 'docs/redesign/phase-9/p9-requal-01/pdf-proof.py'}
selected = [name for name in paths if name.startswith(prefixes) or name in exact
or name.startswith(('Directory.', 'job-tracker-ui/playwright', 'job-tracker-ui/scripts/'))
or name.startswith('docs/cv-redesign/') and name.endswith(('.py', '.ts', '.json'))]
return {name: digest((ROOT / name).read_bytes()) for name in sorted(set(selected)) if (ROOT / name).is_file()}
def tree(path):
return {file.relative_to(path).as_posix(): digest(file.read_bytes()) for file in sorted(path.rglob('*')) if file.is_file()}
def image_id(value):
if not re.fullmatch(r'sha256:[a-f0-9]{64}', value):
raise ValueError('An immutable local image ID is required.')
return value
def schema_name(value):
if not re.fullmatch(r'jt_it_[a-f0-9]{32}', value):
raise ValueError('Only a generated disposable schema is allowed.')
return value
def guard_db(data, name, project, port):
if not re.fullmatch(r'jobtracker-it-[a-f0-9]{32}', project) or name != project + '-mariadb-test-1':
raise ValueError('Unexpected generated DB owner.')
labels = data.get('Config', {}).get('Labels', {})
host = data.get('HostConfig', {})
if (data.get('Name') != '/' + name or data['Config']['Image'] != PIN
or labels.get('com.docker.compose.project') != project
or labels.get('com.docker.compose.service') != 'mariadb-test'
or not data['State']['Running'] or data['State'].get('Health', {}).get('Status') != 'healthy'
or host.get('Privileged') or host.get('NetworkMode') == 'host'
or '/var/lib/mysql' not in host.get('Tmpfs', {})
or host.get('PortBindings', {}).get('3306/tcp') != [{'HostIp': '127.0.0.1', 'HostPort': str(port)}]
or any(mount.get('Destination') == '/var/lib/mysql' and mount.get('Type') != 'tmpfs' for mount in data.get('Mounts', []))):
raise ValueError('DB must be the exact healthy pinned loopback tmpfs test service.')
def verify_build(record, expected_image, current_sources):
if (record.get('exitCode') != 0 or record.get('baseImageId') != BASE
or record.get('imageId') != image_id(expected_image)
or record.get('applicationPath') != APP
or record.get('sourceSha256') != identity(current_sources)
or record.get('sources') != current_sources):
raise ValueError('Current source/build/image identity differs; rebuild in a fresh root.')
def build(args):
out = fresh(args.output)
commands = Commands(out)
record = {'startedUtc': utc(), 'exitCode': 1, 'baseImageId': BASE, 'applicationPath': APP}
try:
record['sources'] = sources(commands)
record['sourceSha256'] = identity(record['sources'])
record['commit'] = commands.run(['git', 'rev-parse', 'HEAD'])
record['status'] = commands.run(['git', 'status', '--short'])
record['dotnetInfo'] = commands.run([args.dotnet, '--info'])
assert commands.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', BASE]) == BASE
record['baseMetadata'] = json.loads(commands.run(['docker', 'image', 'inspect', '--format', '{{json .Config.Labels}}', BASE]))
native_tag = 'jobtracker-cv-ux-native:' + uuid.uuid4().hex
commands.run(['docker', 'tag', BASE, native_tag])
assert commands.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', native_tag]) == BASE
record['baseImageTag'] = native_tag
publish = out / 'publish'
env = {**os.environ, 'DOTNET_ROOT': str(Path(args.dotnet).parent)}
record['publishEnvironment'] = {'DOTNET_ROOT': env['DOTNET_ROOT'], 'NuGetAudit': env.get('NuGetAudit')}
commands.run([args.dotnet, 'publish', 'JobTrackerApi/JobTrackerApi.csproj', '--configuration', 'Release',
'--no-restore', '--output', str(publish), '/p:UseAppHost=false', '--self-contained', 'false'], env=env)
record['publish'] = tree(publish)
record['publishSha256'] = identity(record['publish'])
if 'JobTrackerApi.dll' not in record['publish'] or 'JobTrackerApi.deps.json' not in record['publish']:
raise ValueError('Portable current API output is incomplete.')
dockerfile = out / 'Dockerfile'
dockerfile.write_text(f'FROM {native_tag}\nWORKDIR {APP}\nCOPY publish/ ./\n'
+ f'LABEL io.jobjakt.cv-ux.source-sha256="{record["sourceSha256"]}" '
+ f'io.jobjakt.cv-ux.publish-sha256="{record["publishSha256"]}"\n'
+ 'ENTRYPOINT ["dotnet","JobTrackerApi.dll"]\n', encoding='utf-8')
record['dockerfileSha256'] = digest(dockerfile.read_bytes())
tag = 'jobtracker-cv-ux-api:' + uuid.uuid4().hex
commands.run(['docker', 'build', '--network', 'none', '--pull=false', '--file', str(dockerfile), '--tag', tag, str(out)])
record['imageTag'] = tag
record['imageId'] = image_id(commands.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', tag]))
if record['sources'] != sources(commands) or record['publish'] != tree(publish):
raise ValueError('Source or publish bytes changed during build.')
record['exitCode'] = 0
except Exception as error:
record['error'] = scrub(error)
finally:
record['finishedUtc'] = utc()
write(out / 'build.json', record)
print(json.dumps({'exitCode': record['exitCode'], 'record': str(out / 'build.json')}))
return record['exitCode']
def free_port(port=0):
with socket.socket() as probe:
probe.bind(('127.0.0.1', port))
return probe.getsockname()[1]
def auth_identity(version, commit):
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
request = urllib.request.Request('http://localhost:5302/api/auth/login',
data=json.dumps({'email': 'e2e@example.test', 'password': AUTH_PASSWORD}).encode(),
headers={'Content-Type': 'application/json', 'Origin': 'http://localhost:3300'})
with opener.open(request, timeout=10) as response:
response.read()
with opener.open('http://localhost:5302/api/auth/me', timeout=10) as response:
me = json.load(response)
if me.get('appVersion') != version or me.get('appCommitSha') != commit:
raise ValueError('Authenticated API build metadata mismatch.')
return {'appVersion': me['appVersion'], 'appCommitSha': me['appCommitSha']}
def run(args):
out = fresh(args.output)
commands = Commands(out)
record = {'startedUtc': utc(), 'exitCode': 1, 'cleanup': {}}
suffix = uuid.uuid4().hex
project = 'jobtracker-it-' + suffix
db = project + '-mariadb-test-1'
network = 'jobtracker-cv-ux-' + suffix
api = network + '-api'
schema = schema_name('jt_it_' + uuid.uuid4().hex)
compose = ['docker', 'compose', '--project-name', project, '--project-directory', str(ROOT),
'-f', str(ROOT / 'docker-compose.database.yml'), '--profile', 'integration']
db_attempted = created = attached = network_created = api_created = False
def sql(query):
return commands.run(['docker', 'exec', '-e', 'MYSQL_PWD=' + PASSWORD, db,
'mariadb', '-uroot', '--batch', '--skip-column-names', '-e', query])
def clean(key, action):
try:
action()
record['cleanup'][key] = True
except Exception as error:
record['cleanup'][key + 'Error'] = scrub(error)
record['exitCode'] = 1
try:
build_record = json.loads(Path(args.build_record).read_text(encoding='utf-8'))
current_sources = sources(commands)
verify_build(build_record, args.image, current_sources)
publish = Path(args.build_record).resolve().parent / 'publish'
if tree(publish) != build_record['publish']:
raise ValueError('Published API output drifted after build.')
static = ROOT / 'job-tracker-ui/out'
record['frontend'] = tree(static)
if not record['frontend']:
raise ValueError('Prebuilt frontend output is required.')
record.update({'buildRecord': str(Path(args.build_record).resolve()), 'buildRecordSha256': digest(Path(args.build_record).read_bytes()),
'sourceSha256': build_record['sourceSha256'], 'imageId': args.image, 'commit': build_record['commit'],
'schema': schema, 'dbContainer': db, 'dbProject': project, 'apiContainer': api, 'network': network})
image = json.loads(commands.run(['docker', 'image', 'inspect', args.image]))[0]
if (image['Id'] != args.image or image['Config']['WorkingDir'] != APP
or image['Config'].get('Labels', {}).get('io.jobjakt.cv-ux.source-sha256') != record['sourceSha256']
or image['Config'].get('Labels', {}).get('io.jobjakt.cv-ux.publish-sha256') != build_record['publishSha256']):
raise ValueError('Image source/publish labels or current app directory mismatch.')
free_port(5302)
free_port(3300)
record['chromium'] = commands.run(['docker', 'run', '--rm', '--network', 'none', '--entrypoint', '/usr/bin/chromium', args.image, '--version'])
record['runtime'] = commands.run(['docker', 'run', '--rm', '--network', 'none', '--entrypoint', 'dotnet', args.image, '--info'])
record['apiDllSha256'] = commands.run(['docker', 'run', '--rm', '--network', 'none', '--entrypoint', 'sha256sum', args.image, APP + '/JobTrackerApi.dll']).split()[0]
if record['apiDllSha256'] != build_record['publish']['JobTrackerApi.dll']:
raise ValueError('Image API DLL differs from current publish bytes.')
port = free_port()
record['dbPort'] = port
environment = {**os.environ, 'MARIADB_TEST_PORT': str(port)}
assert commands.run(['docker', 'image', 'inspect', '--format', '{{.Id}}', PIN])
db_attempted = True
commands.run(compose + ['up', '--pull', 'never', '-d', '--wait', '--wait-timeout', '120', 'mariadb-test'], env=environment)
guard_db(json.loads(commands.run(['docker', 'inspect', db]))[0], db, project, port)
record['serverVersion'] = sql('SELECT VERSION()')
if not record['serverVersion'].startswith('12.2.2-MariaDB'):
raise ValueError('Exact MariaDB 12.2.2 is required.')
assert sql(f"SELECT COUNT(*) FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='{schema}'") == '0'
sql(f'CREATE DATABASE `{schema}` CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci')
created = True
# A regular owned bridge permits Docker's loopback-only published API port.
commands.run(['docker', 'network', 'create', '--label', 'task=cv-ux-01', network])
network_created = True
commands.run(['docker', 'network', 'connect', '--alias', 'mariadb', network, db])
attached = True
version = 'cv-ux-linux-' + record['sourceSha256'][:12]
variables = {'ASPNETCORE_ENVIRONMENT': 'Development', 'Database__Provider': 'mariadb', 'Database__ServerVersion': '12.2.2',
'ConnectionStrings__JobTracker': f'Server=mariadb;Database={schema};User ID=root;Password={PASSWORD};SslMode=None',
'Auth__Require': 'true', 'Auth__JwtKey': JWT, 'Auth__AdminEmail': 'e2e@example.test', 'Auth__AdminPassword': AUTH_PASSWORD,
'Auth__AllowRegistration': 'false', 'RateLimits__AuthLoginPermitLimit': '1000', 'Cors__Origins__0': 'http://localhost:3300',
'Workers__AiOperationsEnabled': 'false', 'Workers__SavedJobSearchAlertsEnabled': 'false', 'HttpsRedirection__Enabled': 'false',
'App__Version': version, 'App__CommitSha': record['commit'], 'Data__Root': '/data', 'Logging__LogLevel__Default': 'Warning'}
create = ['docker', 'create', '--name', api, '--label', 'task=cv-ux-01', '--network', network,
'-p', '127.0.0.1:5302:8080', '--tmpfs', '/data:rw,nosuid,size=512m']
for key, value in variables.items():
create += ['-e', key + '=' + value]
api_created = True
commands.run(create + [args.image])
assert commands.run(['docker', 'inspect', '--format', '{{.Image}}', api]) == args.image
commands.run(['docker', 'start', api])
bindings = json.loads(commands.run(['docker', 'inspect', '--format', '{{json .NetworkSettings.Ports}}', api]))
if bindings.get('8080/tcp') != [{'HostIp': '127.0.0.1', 'HostPort': '5302'}]:
raise ValueError('Owned API loopback port was not published.')
deadline = time.monotonic() + 120
while True:
try:
with urllib.request.urlopen('http://localhost:5302/health', timeout=3) as response:
health = json.load(response)
if health.get('version') != version:
raise ValueError('Linux API health version mismatch.')
with urllib.request.urlopen('http://localhost:5302/ready', timeout=3) as response:
assert json.load(response)['status'] == 'ready'
record['health'] = health
record['apiBuildMetadata'] = auth_identity(version, record['commit'])
break
except Exception:
if time.monotonic() >= deadline:
raise
time.sleep(1)
migration_sql = f'SELECT MigrationId FROM `{schema}`.`__EFMigrationsHistory` ORDER BY MigrationId'
record['migrations'] = sql(migration_sql).splitlines()
if '20261025090000_AddCoverLetterDocumentDesign' not in record['migrations']:
raise ValueError('Current letter design migration was not applied.')
env = {**os.environ, 'CV_UX_LINUX_READY': version, 'CV_UX_LINUX_SOURCE': record['sourceSha256'],
'PLAYWRIGHT_STATIC_EXPORT': 'true', 'PLAYWRIGHT_PREBUILT_EXPORT': 'true', 'PLAYWRIGHT_REUSE_SERVERS': 'false',
'NEXT_PUBLIC_API_BASE_URL': 'http://localhost:5302/api', 'CV_FIDELITY_OUTPUT': str(out / 'fidelity'),
'CV_SETTINGS_FIDELITY_OUTPUT': str(out / 'settings-fidelity'), 'LETTER_FIDELITY_OUTPUT': str(out / 'letter-fidelity'),
'PLAYWRIGHT_JSON_OUTPUT_NAME': str(out / 'linux-renderer.json')}
commands.run(['node', 'node_modules/@playwright/test/cli.js', 'test', '--config=' + CONFIG, '--retries=0',
'--reporter=list,json', '--output=' + str(out / 'test-results')], cwd=ROOT / 'job-tracker-ui', env=env, timeout=1800)
native = json.loads((out / 'linux-renderer.json').read_text(encoding='utf-8'))
stats = native.get('stats', {})
if stats.get('expected') != 6 or any(stats.get(key) != 0 for key in ('unexpected', 'flaky', 'skipped')) or native.get('errors'):
raise ValueError('Expected six passing browser cases with no skips, retries or errors.')
record['browserStats'] = stats
if sql(migration_sql).splitlines() != record['migrations'] or sources(commands) != current_sources or tree(static) != record['frontend']:
raise ValueError('Source/runtime/migration drift during rendering.')
for folder in ('fidelity', 'letter-fidelity'):
if not (out / folder / 'measurements.json').is_file():
raise ValueError('Native corpus measurements missing: ' + folder)
cv = json.loads((out / 'fidelity/measurements.json').read_text(encoding='utf-8'))
letter = json.loads((out / 'letter-fidelity/measurements.json').read_text(encoding='utf-8'))
settings_files = list((out / 'settings-fidelity').glob('*-measurements.json'))
if len(cv.get('fixtures', [])) != 16 or len(settings_files) != 4 or sum(len(json.loads(file.read_text(encoding='utf-8')).get('fixtures', [])) for file in settings_files) != 8:
raise ValueError('The original CV16 + settings8 document membership is incomplete.')
if {fixture.get('fixture') for fixture in letter.get('fixtures', [])} != {'light-en-a4', 'dark-nb-a4', 'light-nb-letter', 'dark-en-letter'}:
raise ValueError('The independent letter4 document membership is incomplete.')
record['exitCode'] = 0
except Exception as error:
record['error'] = scrub(error)
finally:
if api_created:
clean('apiLogsCaptured', lambda: commands.run(['docker', 'logs', api]))
clean('apiRemoved', lambda: commands.run(['docker', 'rm', '--force', api]))
if attached:
clean('databaseDisconnected', lambda: commands.run(['docker', 'network', 'disconnect', network, db]))
if network_created:
clean('networkRemoved', lambda: commands.run(['docker', 'network', 'rm', network]))
if created and (not api_created or record['cleanup'].get('apiRemoved')):
def remove_schema():
sql(f'DROP DATABASE `{schema_name(schema)}`')
assert sql(f"SELECT COUNT(*) FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME='{schema}'") == '0'
clean('schemaRemoved', remove_schema)
elif created:
record['cleanup']['schemaRemovedError'] = 'API removal unconfirmed; schema retained.'
record['exitCode'] = 1
if db_attempted:
clean('dbLogsCaptured', lambda: commands.run(compose + ['logs', '--no-color', 'mariadb-test']))
clean('databaseRemoved', lambda: commands.run(compose + ['rm', '--stop', '--force', 'mariadb-test']))
record['finishedUtc'] = utc()
write(out / 'process.json', record)
print(json.dumps({'exitCode': record['exitCode'], 'record': str(out / 'process.json')}))
return record['exitCode']
def main():
parser = argparse.ArgumentParser(description=__doc__)
modes = parser.add_subparsers(dest='mode', required=True)
image = modes.add_parser('build')
image.add_argument('--output', required=True)
image.add_argument('--dotnet', default='C:/Users/Connor/.dotnet/dotnet.exe')
renderer = modes.add_parser('run')
renderer.add_argument('--output', required=True)
renderer.add_argument('--build-record', required=True)
renderer.add_argument('--image', required=True)
args = parser.parse_args()
return build(args) if args.mode == 'build' else run(args)
if __name__ == '__main__':
raise SystemExit(main())