test: qualify immutable models and Phase 9 regression

2026-10-25 13:43:50 +01:00
parent 30e251a593
commit b48f25c4ff
33 changed files with 67462 additions and 6 deletions
@@ -0,0 +1,57 @@
+{
+  "weights-only": {
+    "fixture": "actual-distilbart-cnn-12-6",
+    "sourceRevision": "a4f8f3ea906ed274767e9906dbaede7531d660ff",
+    "license": "Apache-2.0 pinned model card",
+    "files": [
+      {
+        "file": "pytorch_model.bin",
+        "bytes": 1222317369,
+        "sha256": "3bac65d18c99463302d12ca75c2220ea714f9c81ce235f205fa818efe71df6ea",
+        "source": "https://huggingface.co/sshleifer/distilbart-cnn-12-6/resolve/a4f8f3ea906ed274767e9906dbaede7531d660ff/pytorch_model.bin",
+        "license": "Apache-2.0 pinned upstream model card"
+      },
+      {
+        "file": "README.md",
+        "url": "https://huggingface.co/sshleifer/distilbart-cnn-12-6/resolve/a4f8f3ea906ed274767e9906dbaede7531d660ff/README.md",
+        "gitBlobId": "ab24d89a622b44fba741e4383213ad300fc2345a",
+        "bytes": 1705,
+        "sha256": "e6114f9ddddd7aec5e9858b9849d1b53dc42b596420888b82fbf3f8defb64bf7",
+        "license": "Apache-2.0, pinned upstream model-card declaration"
+      },
+      {
+        "file": "config.json",
+        "url": "https://huggingface.co/sshleifer/distilbart-cnn-12-6/resolve/a4f8f3ea906ed274767e9906dbaede7531d660ff/config.json",
+        "gitBlobId": "fade82e22ec4f4b61f6f2eb3986ce4a64a8e23df",
+        "bytes": 1802,
+        "sha256": "028d72a28b90a369eff3f99ceb585fbd2ababf070e8577d72c315757a49b8917",
+        "license": "Apache-2.0, pinned upstream model-card declaration"
+      },
+      {
+        "file": "vocab.json",
+        "url": "https://huggingface.co/sshleifer/distilbart-cnn-12-6/resolve/a4f8f3ea906ed274767e9906dbaede7531d660ff/vocab.json",
+        "gitBlobId": "0a39732b2d8be8e493cab3da68b68cc3e28221de",
+        "bytes": 898822,
+        "sha256": "06b4d46c8e752d410213d9548eb27a54db70fda0319b6271fb8d59dead5e1cab",
+        "license": "Apache-2.0, pinned upstream model-card declaration"
+      },
+      {
+        "file": "merges.txt",
+        "url": "https://huggingface.co/sshleifer/distilbart-cnn-12-6/resolve/a4f8f3ea906ed274767e9906dbaede7531d660ff/merges.txt",
+        "gitBlobId": "226b0752cac7789c48f0cb3ec53eda48b7be36cc",
+        "bytes": 456318,
+        "sha256": "1ce1664773c50f3e0cc8842619a93edc4624525b728b188a9e0be33b7726adc5",
+        "license": "Apache-2.0, pinned upstream model-card declaration"
+      },
+      {
+        "file": "tokenizer_config.json",
+        "url": "https://huggingface.co/sshleifer/distilbart-cnn-12-6/resolve/a4f8f3ea906ed274767e9906dbaede7531d660ff/tokenizer_config.json",
+        "gitBlobId": "be4d21d94f3b4687e5a54d84bf6ab46ed0f8defd",
+        "bytes": 26,
+        "sha256": "5e04eb606e3a1583530a42e36c2a6b6615c86f34fe77e44d9ddeb43ff940931f",
+        "license": "Apache-2.0, pinned upstream model-card declaration"
+      }
+    ],
+    "scope": "Actual trained configured checkpoint. CPU loading/generation/API compatibility only; no human semantic acceptance, GPU, production filesystem/cache or deployment."
+  }
+}
@@ -0,0 +1,9 @@
+{
+  "status": "ASSESSED_NOT_A_FAILURE",
+  "warning": "Please make sure the generation config includes forced_bos_token_id=0",
+  "wheelSha256": "eae143e46d8e521da604bc4fdc45af7dd90e96eacbbc27f68d2754150aa5175a",
+  "member": "transformers/generation/configuration_utils.py",
+  "memberSha256": "fa9997a5334fa132f2911f7518163d963a706a82ade7ebca78afce69bfdc3c39",
+  "source": "if kwargs.get(\"force_bos_token_to_be_generated\", False):\n                self.forced_bos_token_id = self.bos_token_id\n                logger.warning_once(\n                    f\"Please make sure the generation config includes `forced_bos_token_id={self.bos_token_id}`. \"\n                )\n\n        # Validate the values of the attributes\n        self.validate(user_set_attributes=user_set_attributes)\n\n    def __hash__(se",
+  "assessment": "Current generation config explicitly preserves backward compatibility by assigning forced_bos_token_id=bos_token_id(0) for the pinned legacy flag. Pinned original model config already records bos/forced_bos0. Original config bytes are preserved; warning retained in actual log, not suppressed. Real CPU generation passed; no semantic-quality approval."
+}
@@ -0,0 +1,11 @@
+# Actual configured checkpoint follow-up
+
+The small immutable BART fixtures passed library, native CPU, serialization and API compatibility. They do not establish that the actual older configured DistilBART checkpoint/configuration loads under the major Transformers transition. This is the remaining reason for one exact pretrained download; no unrelated or larger generative model is introduced.
+
+The user authorises necessary trusted, licensed, size-checked downloads into an approved nonproduction environment. Local disk has over240GiB free and host memory is about32GiB; qualification is bounded to fourCPU/eightGiB/128PIDs/300seconds with network none, read-only checkpoint and no production/Ollama access. The source is the Apache-2.0 repository at immutable revisiona4f8f3ea906ed274767e9906dbaede7531d660ff. Only the1,222,317,369-byte PyTorch checkpoint with SHA2563bac65d18c99463302d12ca75c2220ea714f9c81ce235f205fa818efe71df6ea is fetched; existing verified tokenizer/config bytes are reused.
+
+The entire size and digest must match before any model load. A partial or invalid download remains failed evidence and is never loaded. The loader is the patched actual sidecar path, with explicit weights_only and no trusted remote code. The test starts only after ordinary, provider/PDF and exclusive15minute qualification has finished. It does not compete with measured workloads.
+
+This is a new follow-up, not a rewrite of the earlier no-full-download checkpoints. Results remain PENDING until actual execution. Full-model compatibility does not establish semantic/C11 acceptance, GPU behaviour, live cache ownership/integrity or deployment readiness.
+
+Execution25October2026: **PASS**. The exact checkpoint size/hash matched; the actual source loader used the default pinned repository revision through a private offline Hub snapshot. CPU model/config/tokenizer loading,1024-token processing, finite logits, generation, authentication and repeat-cache contract passed in27.72seconds with peakRSS2858928KiB. The nonroot, read-only, network-none container exited0 and positive daemon evidence verified removal. The preceding PENDING statement is the preserved pre-execution plan. [validation.json](../validation.json) records the actual successor, manifest hash and remaining scope limits.
@@ -0,0 +1,45 @@
+"""Prepare the reviewed trained checkpoint in a NEW nonproduction directory; never reset caches."""
+from pathlib import Path
+import hashlib
+import json
+import shutil
+import sys
+import time
+import urllib.parse
+import urllib.request
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent/'model'))
+from verify import verify_fixture
+
+
+def prepare(destination):
+    destination=Path(destination)
+    manifest=json.loads(Path(__file__).with_name('ARTIFACTS.json').read_text())['weights-only']
+    revision=manifest['sourceRevision']
+    assert revision=='a4f8f3ea906ed274767e9906dbaede7531d660ff'
+    assert manifest['license'].startswith('Apache-2.0')
+    assert not destination.exists(), 'Use a new isolated directory; existing data is never replaced.'
+    assert shutil.disk_usage(destination.parent).free>8*1024**3, 'Require8GiB available for bounded preparation.'
+    destination.mkdir(mode=0o700)
+    deadline=time.monotonic()+1200
+    for entry in manifest['files']:
+        name=entry['file']; assert Path(name).name==name and name not in ('.','..')
+        url=entry.get('url') or entry['source']
+        parsed=urllib.parse.urlsplit(url)
+        assert parsed.scheme=='https' and parsed.netloc=='huggingface.co'
+        assert parsed.path.startswith('/sshleifer/distilbart-cnn-12-6/resolve/'+revision+'/')
+        temporary=destination/(name+'.partial'); count=0; digest=hashlib.sha256()
+        with urllib.request.urlopen(url,timeout=60) as response, temporary.open('xb') as stream:
+            while block:=response.read(1024*1024):
+                count+=len(block)
+                assert count<=entry['bytes'] and time.monotonic()<deadline, 'Artifact size/deadline exceeded.'
+                digest.update(block); stream.write(block)
+        assert count==entry['bytes'] and digest.hexdigest()==entry['sha256'], 'Artifact identity mismatch; do not load partial files.'
+        temporary.rename(destination/name)
+    verify_fixture(destination,manifest)
+    print(json.dumps({'status':'PASS','directory':str(destination.resolve()),'revision':revision,'files':len(manifest['files'])}))
+
+
+if __name__=='__main__':
+    assert len(sys.argv)==2, 'Usage: python prepare.py NEW_NONPRODUCTION_CACHE_DIRECTORY'
+    prepare(sys.argv[1])
@@ -0,0 +1,84 @@
+"""Actual pinned trained-checkpoint CPU compatibility; human quality remains separate."""
+from pathlib import Path
+import importlib.metadata
+import hashlib
+import json
+import os
+import resource
+import sys
+import time
+sys.path.insert(0, '/qualification')
+from verify import verify_fixture
+
+HERE = Path(__file__).resolve().parent
+manifest_bytes = (HERE / 'ARTIFACTS.json').read_bytes()
+manifests = json.loads(manifest_bytes)
+manifest_sha256 = hashlib.sha256(manifest_bytes).hexdigest()
+kind = sys.argv[1]
+fixture = verify_fixture(Path(sys.argv[2]), manifests[kind])
+assert os.environ['HF_HUB_OFFLINE'] == os.environ['TRANSFORMERS_OFFLINE'] == '1'
+manifest = manifests[kind]
+repository_name = 'sshleifer/distilbart-cnn-12-6'
+assert manifest['fixture'] == 'actual-distilbart-cnn-12-6'
+cache_home = Path('/tmp/qualified-hf-cache')
+cache_home.mkdir(mode=0o700)
+os.environ['HF_HOME'] = str(cache_home)
+cache = cache_home / 'hub'
+repository = cache / ('models--' + repository_name.replace('/', '--'))
+snapshot = repository / 'snapshots' / manifest['sourceRevision']
+snapshot.mkdir(parents=True)
+(repository / 'blobs').mkdir()
+for entry in manifest['files']:
+    identity = entry.get('gitBlobId') or entry['sha256']
+    blob = repository / 'blobs' / identity
+    blob.symlink_to(fixture / entry['file'])
+    (snapshot / entry['file']).symlink_to(Path('../../blobs') / identity)
+os.environ.pop('AI_SERVICE_SKIP_MODEL_LOAD', None)
+os.environ.pop('AI_SERVICE_EAGER_MODEL_LOAD', None)
+os.environ['AI_SERVICE_TOKEN'] = 'p9-local-qualification-only'
+sys.path.insert(0, '/app')
+import torch
+import app
+from fastapi.testclient import TestClient
+
+torch.set_num_threads(1)
+started = time.monotonic()
+from huggingface_hub import try_to_load_from_cache
+assert app.MODEL_NAME == repository_name
+assert app.MODEL_REVISION == manifest['sourceRevision']
+for entry in manifest['files']:
+    cached = try_to_load_from_cache(app.MODEL_NAME, entry['file'], revision=app.MODEL_REVISION)
+    assert isinstance(cached, str) and Path(cached).resolve() == fixture / entry['file']
+assert try_to_load_from_cache(app.MODEL_NAME, 'absent-qualification-file', revision=app.MODEL_REVISION) is None
+assert app._ensure_runtime_loaded(), app.MODEL_LOAD_ERROR
+assert app.model.__class__.__name__ == 'BartForConditionalGeneration'
+assert (app.model.config.d_model, app.model.config.encoder_layers, app.model.config.decoder_layers, app.model.config.vocab_size) == (1024, 12, 6, 50264)
+assert len(app.tokenizer) == 50265
+assert app.model.training is False
+assert str(app.device) == 'cpu' and not app.GPU_AVAILABLE
+text = 'Åse Østby builds accessible Python and C# applications for reliable Norwegian career workflows.'
+encoded = app.tokenizer(text, return_tensors='pt')
+assert app.tokenizer.decode(encoded.input_ids[0], skip_special_tokens=True) == text
+summary = app._model_summarize(text, 24, 8)
+assert isinstance(summary, str) and summary
+long_input = app.tokenizer(text * 400, return_tensors='pt', truncation=True, max_length=1024)
+assert long_input.input_ids.shape == (1, 1024)
+assert long_input.input_ids.dtype == torch.int64
+with torch.no_grad():
+    logits = app.model(**long_input, decoder_input_ids=torch.tensor([[2, 0, 10]])).logits
+    assert torch.isfinite(logits).all()
+assert app._model_summarize(text * 400, 24, 8)
+client = TestClient(app.app)
+assert client.get('/health').json()['model_loaded'] is True
+assert client.post('/summarize', json={'text': text}).status_code == 401
+headers = {'X-Ai-Service-Token': 'p9-local-qualification-only'}
+body = {'text': text, 'max_length': 24, 'min_length': 8}
+first = client.post('/summarize', json=body, headers=headers)
+second = client.post('/summarize', json=body, headers=headers)
+assert first.status_code == second.status_code == 200, first.text
+assert first.json()['cached'] is False and second.json()['cached'] is True
+assert first.json()['summary'] == second.json()['summary']
+verify_fixture(fixture, manifests[kind])
+elapsed = time.monotonic() - started
+assert elapsed < 240
+print(json.dumps({'status': 'PASS', 'kind': kind, 'scope': 'Actual pinned trained DistilBART CPU loading/tokenizer/generation and summary/auth/cache compatibility; no human semantic-quality/GPU/production claim.', 'manifestSha256': manifest_sha256, 'repositoryRevision': app.MODEL_REVISION, 'modelClass': app.model.__class__.__name__, 'device': str(app.device), 'cudaAvailable': torch.cuda.is_available(), 'seconds': elapsed, 'maxRssKiB': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, 'versions': {name: importlib.metadata.version(name) for name in ['torch','transformers','filelock','huggingface-hub','tokenizers','safetensors']}, 'summary': summary, 'bounded1024Tokens': True, 'finiteLogits': True, 'cacheRepeatedRead': True, 'fixtureUnchanged': True}), flush=True)
@@ -0,0 +1,35 @@
+import hashlib
+import io
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+from prepare import prepare
+
+
+class PreparationBoundaryTests(unittest.TestCase):
+    def run_fixture(self, payload, expected=b'approved'):
+        revision='a4f8f3ea906ed274767e9906dbaede7531d660ff'
+        manifest={'weights-only':{'sourceRevision':revision,'license':'Apache-2.0 pinned card',
+            'files':[{'file':'config.json','url':'https://huggingface.co/sshleifer/distilbart-cnn-12-6/resolve/'+revision+'/config.json',
+                      'bytes':len(expected),'sha256':hashlib.sha256(expected).hexdigest()}]}}
+        with tempfile.TemporaryDirectory() as directory:
+            target=Path(directory)/'new-cache'
+            with patch.object(Path,'read_text',return_value=json.dumps(manifest)), patch('prepare.shutil.disk_usage',return_value=type('Usage',(),{'free':10*1024**3})()), patch('prepare.urllib.request.urlopen',return_value=io.BytesIO(payload)):
+                if payload==expected:
+                    prepare(target)
+                    self.assertEqual((target/'config.json').read_bytes(),expected)
+                    with self.assertRaises(AssertionError):prepare(target)
+                    self.assertEqual((target/'config.json').read_bytes(),expected)
+                else:
+                    with self.assertRaises(AssertionError):prepare(target)
+                    self.assertFalse((target/'config.json').exists())
+                    self.assertTrue((target/'config.json.partial').exists())
+
+    def test_validated_bytes_and_existing_cache_preserved(self):self.run_fixture(b'approved')
+    def test_corrupt_bytes_never_promoted(self):self.run_fixture(b'rejected')
+    def test_oversized_bytes_never_promoted(self):self.run_fixture(b'oversized payload')
+
+
+if __name__=='__main__':unittest.main()