test: qualify immutable models and Phase 9 regression

docs/redesign/phase-9/p9-sec-02/model/run.py

@@ -6,6 +6,7 @@ import subprocess
 import sys
 import time
 import uuid
+from cleanup import container_absent
 
 ROOT=Path(__file__).resolve().parents[5]
 HERE=Path(__file__).resolve().parent
@@ -37,14 +38,21 @@ with (out/'process.log').open('w',encoding='utf-8') as log:
     try:code=subprocess.run(['rtk','proxy',*args],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT,timeout=180).returncode
     except subprocess.TimeoutExpired:
         code=124
-        check=subprocess.run(['rtk','proxy','docker','inspect','--format','{{index .Config.Labels "jobtracker.qualification"}}',name],capture_output=True,text=True)
-        if check.returncode==0 and check.stdout.strip()=='p9-sec02':subprocess.run(['rtk','proxy','docker','kill',name],capture_output=True)
+        try:
+            check=subprocess.run(['rtk','proxy','docker','inspect','--format','{{index .Config.Labels "jobtracker.qualification"}}',name],capture_output=True,text=True,timeout=5)
+            if check.returncode==0 and check.stdout.strip()=='p9-sec02':subprocess.run(['rtk','proxy','docker','kill',name],capture_output=True,timeout=10)
+        except (OSError,subprocess.TimeoutExpired):pass
 deadline=time.monotonic()+5
+cleanup_error=None
 while True:
-    absent=subprocess.run(['rtk','proxy','docker','inspect',name],capture_output=True).returncode!=0
+    try:absent=container_absent(name)
+    except RuntimeError as error:
+        absent=None;cleanup_error=str(error);break
     if absent or time.monotonic()>=deadline:break
     time.sleep(.1)
 record={'lane':lane,'attempt':attempt,'startedUtc':started,'finishedUtc':datetime.datetime.now(datetime.timezone.utc).isoformat(),'exitCode':code,'command':args,'localImageId':image,'runtimeImageId':runtime['runtimeImageId'],'containerAbsent':absent,'network':'none','readOnly':True,'user':'65534:65534','memoryLimitBytes':2147483648,'cpuLimit':2,'pidsLimit':128,'wallClockDeadlineSeconds':180,'scope':'Actual locally built source image plus exact test-only extras; no app source mount, production cache, external network, registry push or deployment.'}
+record['cleanupStatus']='ABSENT' if absent else 'UNKNOWN' if absent is None else 'PRESENT'
+record['cleanupError']=cleanup_error
 (out/'process.json').write_text(json.dumps(record,indent=2)+'\n')
 print(json.dumps(record))
 print((out/'process.log').read_text(encoding='utf-8',errors='replace')[-2500:].encode('ascii','replace').decode())