From 281d1bdb071813fcfefe1547628556234f26f950 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 1 Oct 2026 20:57:16 -0500 Subject: [PATCH 1/2] ci(GH-992): retain sanitized full smoke diagnostics Record startup, service and artifact identities before private cleanup. Refs QRun-IO/qqq#992; original hosted failure remains unexplained. --- .github/workflows/smoke.yml | 13 ++- scripts/run_full_smoke.py | 29 +++-- scripts/smoke_diagnostics.py | 139 +++++++++++++++++++++++ scripts/test_smoke_diagnostics.py | 181 ++++++++++++++++++++++++++++++ 4 files changed, 354 insertions(+), 8 deletions(-) create mode 100644 scripts/smoke_diagnostics.py create mode 100644 scripts/test_smoke_diagnostics.py diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 28a8a26..c0d3730 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -42,6 +42,17 @@ jobs: java-version: "21" cache: maven - name: Package full app for Docker image - run: mvn -B -ntp -pl qqq-all-app -am -DskipTests package + run: >- + mvn -B -ntp -pl qqq-all-app -am -DskipTests + dependency:build-classpath -DincludeGroupIds=com.kingsrook.qqq,com.kingsrook.qbits + -Dmdep.outputFile=target/smoke-classpath.txt package - name: Smoke full Compose over HTTP run: python3 scripts/run_full_smoke.py + - name: Retain sanitized full smoke diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: full-smoke-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: target/smoke-diagnostics/*.json + if-no-files-found: warn + retention-days: 14 diff --git a/scripts/run_full_smoke.py b/scripts/run_full_smoke.py index 138621e..8543c66 100644 --- a/scripts/run_full_smoke.py +++ b/scripts/run_full_smoke.py @@ -12,6 +12,7 @@ import urllib.parse from pathlib import Path from smoke import check_full +from smoke_diagnostics import Diagnostics ROOT = Path(__file__).resolve().parent.parent @@ -95,6 +96,7 @@ def main(): environment = os.environ.copy() environment.update(values) os.environ.update(values) + diagnostics = Diagnostics(ROOT / "target/smoke-diagnostics", environment) with tempfile.TemporaryDirectory(prefix="qqq-all-full-smoke-") as directory: env_file = Path(directory) / "compose.env" env_file.write_text("".join(f"{key}={value}\n" for key, value in values.items())) @@ -103,10 +105,11 @@ def main(): "-f", str(ROOT / "compose.yaml"), "-f", str(ROOT / "scripts/compose.smoke.yaml"), "--profile", "full"] try: - started = subprocess.run(command + ["up", "-d", "--build", "--wait", "--wait-timeout", "360"], - cwd=ROOT, env=environment, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, timeout=480, check=False) - if started.returncode: + diagnostics.jar_hashes(ROOT) + started = diagnostics.run(command + ["up", "-d", "--build", "--wait", "--wait-timeout", "360"], + ROOT, environment, timeout=480) + diagnostics.write("startup.json", started) + if started["exit_code"] != 0: raise AssertionError("full Compose stack did not become healthy") prepare_sftp_import_directory(command, environment, values.get("SFTP_USER", "qqq")) check_full( @@ -115,10 +118,22 @@ def main(): lambda index, query: query_opensearch(command, environment, index, query), 120, ) + except (AssertionError, OSError, ValueError, subprocess.TimeoutExpired) as error: + diagnostics.write("failure.json", {"type": type(error).__name__, + "message": diagnostics.sanitize(str(error))}) + raise AssertionError(diagnostics.sanitize(str(error))) from None finally: - subprocess.run(command + ["down", "--volumes", "--remove-orphans"], - cwd=ROOT, env=environment, stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, timeout=120, check=False) + try: + diagnostics.collect(command, ROOT, environment) + except Exception as error: + # Evidence failure must not suppress the smoke error or private-stack cleanup. + print(f"smoke diagnostics collection failed: {type(error).__name__}", file=sys.stderr) + finally: + cleanup = diagnostics.run(command + ["down", "--volumes", "--remove-orphans"], + ROOT, environment, timeout=120) + diagnostics.write("cleanup.json", cleanup) + if cleanup["exit_code"] != 0 and sys.exc_info()[0] is None: + raise AssertionError("private Compose cleanup failed; see sanitized diagnostics") if __name__ == "__main__": diff --git a/scripts/smoke_diagnostics.py b/scripts/smoke_diagnostics.py new file mode 100644 index 0000000..7b7d300 --- /dev/null +++ b/scripts/smoke_diagnostics.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +"""Small, sanitized receipts for the private full-smoke stack only.""" + +import base64 +import hashlib +import json +import os +import re +import subprocess +import sys +import time +import urllib.parse +from pathlib import Path + + +LIMIT = 65536 +CONTAINER_FORMAT = ('{"id":{{json .Id}},"service":{{json (index .Config.Labels ' + '"com.docker.compose.service")}},"image_id":{{json .Image}},' + '"status":{{json .State.Status}},"exit_code":{{json .State.ExitCode}},' + '"oom_killed":{{json .State.OOMKilled}},"error":{{json .State.Error}},' + '"health":{{if index .State "Health"}}{{json (index .State "Health")}}{{else}}null{{end}}}') +IMAGE_FORMAT = ('{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},' + '"architecture":{{json .Architecture}},"os":{{json .Os}}}') + + +class Diagnostics: + def __init__(self, directory, environment): + self.directory = directory + self.secrets = set() + for key, value in environment.items(): + if value and re.search(r'PASSWORD|PASSWD|SECRET|TOKEN|KEYFILE|ACCESS_KEY|PRIVATE_KEY|CREDENTIAL|AUTHORIZATION', key, re.I): + self.secrets.update((value, urllib.parse.quote(value, safe=''), + urllib.parse.quote_plus(value), + base64.b64encode(value.encode()).decode())) + + def sanitize(self, text): + if isinstance(text, bytes): + text = text.decode('utf-8', errors='replace') + text = text or '' + text = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', text) + for secret in sorted(self.secrets, key=len, reverse=True): + text = text.replace(secret, '[REDACTED]') + text = re.sub(r'(?i)(\b[\w.-]{0,80}(?:password|passwd|secret|token|authorization|credential|api[_-]?key|access[_-]?key)\b["\s]*[:=]\s*)("[^"]*"|\x27[^\x27]*\x27|[^\s,;]+)', r'\1[REDACTED]', text) + text = re.sub(r'(?i)\b(Bearer|Basic)\s+[^\s,;]+', r'\1 [REDACTED]', text) + text = re.sub(r'(?i)\b([a-z][a-z0-9+.-]*://)[^/\s@]+@', r'\1[REDACTED]@', text) + if len(text) > LIMIT: + text = '[...truncated...]\n' + text[-(LIMIT - 18):] + return text + + def write(self, name, data): + # Redact strings before JSON encoding, so escaped secrets cannot escape filtering. + def clean(value): + if isinstance(value, str): + return self.sanitize(value) + if isinstance(value, dict): + return {key: clean(item) for key, item in value.items()} + if isinstance(value, list): + return [clean(item) for item in value] + return value + try: + self.directory.mkdir(parents=True, exist_ok=True) + path = self.directory / name + path.write_text(json.dumps(clean(data), indent=2) + '\n') + path.chmod(0o600) + except OSError as error: + print(f"smoke diagnostics write failed: {type(error).__name__}", file=sys.stderr) + + def run(self, args, root, environment, timeout=15): + try: + result = subprocess.run(args, cwd=root, env=environment, capture_output=True, + text=True, timeout=timeout, check=False) + return {'exit_code': result.returncode, + 'output': self.sanitize((result.stdout or '') + (result.stderr or ''))} + except subprocess.TimeoutExpired as error: + return {'exit_code': None, 'error': 'TimeoutExpired', + 'output': self.sanitize(self.sanitize(error.stdout) + self.sanitize(error.stderr))} + except OSError as error: + return {'exit_code': None, 'error': type(error).__name__, 'output': ''} + + def jar_hashes(self, root): + data = {'dependencies': []} + manifest = root / 'qqq-all-app/target/smoke-classpath.txt' + if manifest.is_file(): + for name in sorted(set(manifest.read_text().strip().split(os.pathsep))): + path = Path(name) + match = re.search(r'/com/kingsrook/(qqq|qbits)/([^/]+)/([^/]+)/[^/]+\.jar$', path.as_posix()) + if not match: + continue + group, artifact, version = match.groups() + entry = {'coordinate': f'com.kingsrook.{group}:{artifact}:{version}', 'file': path.name} + if path.is_file(): + with path.open('rb') as stream: + entry['sha256'] = hashlib.file_digest(stream, 'sha256').hexdigest() + else: + entry['error'] = 'resolved JAR missing' + data['dependencies'].append(entry) + else: + data['error'] = 'resolved classpath missing; run the full-smoke Maven command' + app = root / 'qqq-all-app/target/qqq-all-app.jar' + if app.is_file(): + with app.open('rb') as stream: + data['application_sha256'] = hashlib.file_digest(stream, 'sha256').hexdigest() + self.write('jars.json', data) + + def collect(self, command, root, environment): + deadline = time.monotonic() + 60 + def run(args): + remaining = deadline - time.monotonic() + if remaining <= 0: + return {'exit_code': None, 'error': 'collection budget exhausted', 'output': ''} + return self.run(args, root, environment, timeout=min(15, remaining)) + inventory = run(command + ['ps', '-a', '-q']) + ids = [line for line in inventory['output'].splitlines() + if re.fullmatch(r'[a-f0-9]{12,64}', line)][:32] + services, images = [], {} + for container in ids: + result = run(['docker', 'inspect', '--format', CONTAINER_FORMAT, container]) + try: + raw = json.loads(result['output']) + state = {key: raw.get(key) for key in ('id', 'service', 'image_id', 'status', + 'exit_code', 'oom_killed', 'error')} + health = raw.get('health') or {} + state['health'] = {'status': health.get('Status'), + 'checks': [{key: check.get(key) for key in ('Start', 'End', 'ExitCode', 'Output')} + for check in health.get('Log', [])[-5:]]} + services.append(state) + image = state['image_id'] + if isinstance(image, str) and re.fullmatch(r'sha256:[a-f0-9]{64}', image) and image not in images: + found = run(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image]) + try: + details = json.loads(found['output']) + images[image] = {key: details.get(key) for key in ('id', 'repo_digests', 'architecture', 'os')} + except (ValueError, TypeError, AttributeError): + images[image] = {'error': 'image identity unavailable', 'command': found} + except (ValueError, TypeError, AttributeError): + services.append({'id': container, 'error': 'container state unavailable', 'command': result}) + self.write(f'container-{container[:12]}.json', + run(['docker', 'logs', '--timestamps', '--tail', '100', container])) + self.write('services.json', {'inventory': inventory, 'services': services, 'images': images}) diff --git a/scripts/test_smoke_diagnostics.py b/scripts/test_smoke_diagnostics.py new file mode 100644 index 0000000..dafaaf7 --- /dev/null +++ b/scripts/test_smoke_diagnostics.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""Failure evidence must survive cleanup without leaking runtime credentials.""" + +import hashlib +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import run_full_smoke + + +class DiagnosticsTest(unittest.TestCase): + def test_startup_failure_retains_redacted_evidence_before_cleanup(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'compose.yaml').touch() + values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002', + 'POSTGRES_PASSWORD': 'private-password-123'} + cleaned = [] + + def run(args, **kwargs): + if 'up' in args: + return subprocess.CompletedProcess(args, 1, 'postgres unhealthy private-password-123', '') + if 'down' in args: + receipt = root / 'target/smoke-diagnostics/startup.json' + self.assertTrue(receipt.is_file(), 'diagnostics must exist before teardown') + cleaned.append(True) + return subprocess.CompletedProcess(args, 0, '', '') + + with (mock.patch.object(run_full_smoke, 'ROOT', root), + mock.patch.object(run_full_smoke, 'demo_environment', return_value=values), + mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run), + mock.patch.dict(os.environ)): + with self.assertRaisesRegex(AssertionError, 'did not become healthy'): + run_full_smoke.main() + self.assertTrue(cleaned) + output = (root / 'target/smoke-diagnostics/startup.json').read_text() + self.assertIn('postgres unhealthy', output) + self.assertNotIn('private-password-123', output) + self.assertEqual(1, json.loads(output)['exit_code']) + + def test_http_failure_stays_sanitized_when_collection_and_cleanup_fail(self): + self.diagnostics_module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'compose.yaml').touch() + values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002', + 'DB_PASSWORD': 'hidden-fixture'} + def run(args, **kwargs): + if 'down' in args: + raise subprocess.TimeoutExpired(args, 120, output='hidden-fixture') + return subprocess.CompletedProcess(args, 0, '', '') + with (mock.patch.object(run_full_smoke, 'ROOT', root), + mock.patch.object(run_full_smoke, 'demo_environment', return_value=values), + mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run), + mock.patch.object(run_full_smoke.Diagnostics, 'collect', side_effect=ValueError('hidden-fixture')), + mock.patch.object(run_full_smoke, 'check_full', side_effect=AssertionError('HTTP failed hidden-fixture')), + mock.patch.dict(os.environ)): + with self.assertRaisesRegex(AssertionError, r'HTTP failed \[REDACTED\]'): + run_full_smoke.main() + text = ''.join(f.read_text() for f in (root / 'target/smoke-diagnostics').glob('*.json')) + self.assertNotIn('hidden-fixture', text) + self.assertIn('TimeoutExpired', text) + + def test_startup_timeout_keeps_partial_sanitized_output_and_tears_down(self): + self.diagnostics_module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'compose.yaml').touch() + values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002', + 'DB_PASSWORD': 'hidden-fixture'} + cleanup = [] + def run(args, **kwargs): + if 'up' in args: + raise subprocess.TimeoutExpired(args, 480, output=b'postgres unhealthy hidden-fixture') + if 'down' in args: + cleanup.append(True) + return subprocess.CompletedProcess(args, 0, '', '') + with (mock.patch.object(run_full_smoke, 'ROOT', root), + mock.patch.object(run_full_smoke, 'demo_environment', return_value=values), + mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run), + mock.patch.dict(os.environ)): + with self.assertRaisesRegex(AssertionError, 'did not become healthy'): + run_full_smoke.main() + self.assertTrue(cleanup) + receipt = json.loads((root / 'target/smoke-diagnostics/startup.json').read_text()) + self.assertEqual('TimeoutExpired', receipt['error']) + self.assertIn('postgres unhealthy', receipt['output']) + self.assertNotIn('hidden-fixture', receipt['output']) + + def diagnostics_module(self): + try: + import smoke_diagnostics + except ImportError: + self.fail('sanitized diagnostics collector is not implemented') + return smoke_diagnostics + + def test_redacts_secret_forms_and_bounds_output_after_redaction(self): + module = self.diagnostics_module() + with tempfile.TemporaryDirectory() as directory: + evidence = module.Diagnostics(Path(directory), {'DB_PASSWORD': 'a+/secret', + 'GITHUB_TOKEN': 'token-1234'}) + output = evidence.sanitize('x' * 90000 + '\npassword="unknown-value" ' + 'postgres://u:uri-secret@db/ Bearer bearer-secret ' + 'a%2B%2Fsecret a+/secret token-1234 access_token=generated-token client_secret=generated-secret') + for secret in ['unknown-value', 'uri-secret', 'bearer-secret', + 'a%2B%2Fsecret', 'a+/secret', 'token-1234', 'generated-token', 'generated-secret']: + self.assertFalse(secret in output, "credential remained in sanitized output") + self.assertLessEqual(len(output), 65536) + self.assertIn('[REDACTED]', output) + + def test_hashes_only_resolved_first_party_jars_not_other_cached_versions(self): + module = self.diagnostics_module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + jar = root / 'repository/com/kingsrook/qqq/qqq-esb/4.1.0-SNAPSHOT/qqq-esb.jar' + jar.parent.mkdir(parents=True) + jar.write_bytes(b'actual resolved bytes') + (jar.parent / 'unused.jar').write_bytes(b'unused') + manifest = root / 'qqq-all-app/target/smoke-classpath.txt' + manifest.parent.mkdir(parents=True) + manifest.write_text(str(jar)) + evidence = module.Diagnostics(root / 'target/smoke-diagnostics', {}) + evidence.jar_hashes(root) + data = json.loads((evidence.directory / 'jars.json').read_text()) + self.assertEqual([{'coordinate': 'com.kingsrook.qqq:qqq-esb:4.1.0-SNAPSHOT', + 'file': 'qqq-esb.jar', + 'sha256': hashlib.sha256(b'actual resolved bytes').hexdigest()}], data['dependencies']) + self.assertNotIn(str(root), json.dumps(data)) + + def test_container_collection_is_selected_redacted_and_survives_command_failure(self): + module = self.diagnostics_module() + with tempfile.TemporaryDirectory() as directory: + evidence = module.Diagnostics(Path(directory), {'DB_PASSWORD': 'secret-fixture'}) + container = 'a' * 64 + completed = 'd' * 64 + image = 'sha256:' + 'b' * 64 + commands = [] + def run(args, **kwargs): + commands.append(args) + if 'ps' in args: + return subprocess.CompletedProcess(args, 0, container + '\n' + completed + '\n', '') + if 'inspect' in args and 'image' not in args: + self.assertIn('--format', args) + if args[-1] == completed: + return subprocess.CompletedProcess(args, 0, json.dumps({ + 'id': completed, 'service': 'mongo-init', 'image_id': image, + 'status': 'exited', 'exit_code': 0, 'oom_killed': False, + 'error': '', 'health': None}), '') + return subprocess.CompletedProcess(args, 0, json.dumps({ + 'id': container, 'service': 'postgres', 'image_id': image, + 'status': 'exited', 'exit_code': 1, 'oom_killed': False, + 'error': '', 'health': {'Status': 'unhealthy', 'Log': [ + {'ExitCode': 1, 'Output': 'failed secret-fixture'}]}}), '') + if 'image' in args: + return subprocess.CompletedProcess(args, 0, json.dumps({'id': image, + 'repo_digests': ['postgres@sha256:' + 'c' * 64], + 'architecture': 'amd64', 'os': 'linux'}), '') + if 'logs' in args: + raise subprocess.TimeoutExpired(args, 15, output='secret-fixture') + self.fail('unexpected command') + with mock.patch.object(module.subprocess, 'run', side_effect=run): + evidence.collect(['docker', 'compose', '-p', 'private'], Path(directory), {}) + text = '\n'.join(f.read_text() for f in Path(directory).glob('*.json')) + self.assertNotIn('secret-fixture', text) + self.assertIn('unhealthy', text) + self.assertIn('amd64', text) + states = json.loads((Path(directory) / 'services.json').read_text())['services'] + self.assertEqual('mongo-init', states[1]['service']) + self.assertEqual(0, states[1]['exit_code']) + self.assertIsNone(states[1]['health']['status']) + self.assertIn('TimeoutExpired', text) + self.assertFalse(any('config' in c or '--volumes' in c for c in commands)) + + +if __name__ == '__main__': + unittest.main() -- 2.54.0 From cedc3023c8eb6fa59d213a25dd9291c72e97b119 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 1 Oct 2026 21:22:00 -0500 Subject: [PATCH 2/2] ci(GH-992): identify failures in sequential image pulls Retain each selected service, image and sanitized pull result before full startup. Refs QRun-IO/qqq#992. --- scripts/run_full_smoke.py | 1 + scripts/smoke_diagnostics.py | 38 ++++++++++++++++ scripts/test_smoke.py | 1 + scripts/test_smoke_diagnostics.py | 74 +++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+) diff --git a/scripts/run_full_smoke.py b/scripts/run_full_smoke.py index 8543c66..aae8496 100644 --- a/scripts/run_full_smoke.py +++ b/scripts/run_full_smoke.py @@ -106,6 +106,7 @@ def main(): "--profile", "full"] try: diagnostics.jar_hashes(ROOT) + diagnostics.pull(command, ROOT, environment) started = diagnostics.run(command + ["up", "-d", "--build", "--wait", "--wait-timeout", "360"], ROOT, environment, timeout=480) diagnostics.write("startup.json", started) diff --git a/scripts/smoke_diagnostics.py b/scripts/smoke_diagnostics.py index 7b7d300..93715cd 100644 --- a/scripts/smoke_diagnostics.py +++ b/scripts/smoke_diagnostics.py @@ -102,6 +102,44 @@ class Diagnostics: data['application_sha256'] = hashlib.file_digest(stream, 'sha256').hexdigest() self.write('jars.json', data) + def pull(self, command, root, environment): + receipt = {'pulls': []} + try: + # Keep the uninterpolated model in memory only; retain no service environment. + model = subprocess.run(command + ['config', '--format', 'json', '--no-interpolate', + '--no-env-resolution'], cwd=root, env=environment, + capture_output=True, text=True, timeout=15, check=False) + if model.returncode: + raise ValueError('Compose image selection failed') + services = json.loads(model.stdout)['services'] + if not isinstance(services, dict) or not 1 <= len(services) <= 32: + raise ValueError('unexpected service inventory') + selected = [(name, service['image']) for name, service in sorted(services.items()) + if 'build' not in service] + if any(not re.fullmatch(r'[a-zA-Z0-9][a-zA-Z0-9_.-]*', name) + or not isinstance(image, str) or '${' in image for name, image in selected): + raise ValueError('unresolved service image') + except (KeyError, TypeError, ValueError, OSError, subprocess.TimeoutExpired) as error: + receipt['selection_error'] = type(error).__name__ + self.write('pulls.json', receipt) + raise AssertionError('could not select Compose service images; see pulls.json') from None + deadline = time.monotonic() + 480 + for service, image in selected: + first = image.split('/')[0] + registry = first if '/' in image and ('.' in first or ':' in first or first == 'localhost') else 'docker.io' + remaining = deadline - time.monotonic() + if remaining <= 0: + result = {'exit_code': None, 'error': 'pull-stage budget exhausted', 'output': ''} + else: + result = self.run(command + ['--parallel', '1', '--progress', 'plain', 'pull', + '--ignore-buildable', '--policy', 'missing', service], + root, environment, timeout=min(120, remaining)) + receipt['pulls'].append({'service': service, 'image': image, + 'registry': registry, 'result': result}) + self.write('pulls.json', receipt) + if result['exit_code'] != 0: + raise AssertionError(f'Compose pull failed for {service} ({image}); see pulls.json') + def collect(self, command, root, environment): deadline = time.monotonic() + 60 def run(args): diff --git a/scripts/test_smoke.py b/scripts/test_smoke.py index b17069b..fefc29c 100644 --- a/scripts/test_smoke.py +++ b/scripts/test_smoke.py @@ -192,6 +192,7 @@ class SmokeTest(unittest.TestCase): with (mock.patch.object(run_full_smoke, "ROOT", root), mock.patch.object(run_full_smoke, "demo_environment", return_value=values), mock.patch.object(run_full_smoke, "published_port", return_value=20003), + mock.patch.object(run_full_smoke.Diagnostics, "pull"), mock.patch.object(run_full_smoke.subprocess, "run", return_value=subprocess.CompletedProcess([], 0)), mock.patch.object(run_full_smoke, "check_full", side_effect=verify_credentials) as check, diff --git a/scripts/test_smoke_diagnostics.py b/scripts/test_smoke_diagnostics.py index dafaaf7..5f334d5 100644 --- a/scripts/test_smoke_diagnostics.py +++ b/scripts/test_smoke_diagnostics.py @@ -23,6 +23,8 @@ class DiagnosticsTest(unittest.TestCase): cleaned = [] def run(args, **kwargs): + if 'config' in args: + return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '') if 'up' in args: return subprocess.CompletedProcess(args, 1, 'postgres unhealthy private-password-123', '') if 'down' in args: @@ -51,6 +53,8 @@ class DiagnosticsTest(unittest.TestCase): values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002', 'DB_PASSWORD': 'hidden-fixture'} def run(args, **kwargs): + if 'config' in args: + return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '') if 'down' in args: raise subprocess.TimeoutExpired(args, 120, output='hidden-fixture') return subprocess.CompletedProcess(args, 0, '', '') @@ -75,6 +79,8 @@ class DiagnosticsTest(unittest.TestCase): 'DB_PASSWORD': 'hidden-fixture'} cleanup = [] def run(args, **kwargs): + if 'config' in args: + return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '') if 'up' in args: raise subprocess.TimeoutExpired(args, 480, output=b'postgres unhealthy hidden-fixture') if 'down' in args: @@ -92,6 +98,74 @@ class DiagnosticsTest(unittest.TestCase): self.assertIn('postgres unhealthy', receipt['output']) self.assertNotIn('hidden-fixture', receipt['output']) + def test_pull_failure_names_service_image_registry_and_stops_before_up(self): + module = self.diagnostics_module() + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'compose.yaml').touch() + values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002', + 'DB_PASSWORD': 'pull-secret'} + calls = [] + model = {'services': { + 'app-full': {'image': 'qqq-all:local', 'build': {'context': '.'}}, + 'minio-init': {'image': 'public.ecr.aws/aws-cli/aws-cli:2.27.41', + 'environment': {'DO_NOT_RETAIN': 'raw-config-fixture'}}, + 'sftp': {'image': 'atmoz/sftp:alpine'}}} + def run(args, **kwargs): + calls.append(args) + if 'config' in args: + self.assertIn('--no-interpolate', args) + self.assertIn('--no-env-resolution', args) + return subprocess.CompletedProcess(args, 0, json.dumps(model), '') + if 'pull' in args: + self.assertEqual('minio-init', args[-1]) + self.assertNotIn('--ignore-pull-failures', args) + self.assertNotIn('--include-deps', args) + return subprocess.CompletedProcess(args, 1, '', 'toomanyrequests: Rate exceeded pull-secret') + return subprocess.CompletedProcess(args, 0, '', '') + with (mock.patch.object(run_full_smoke, 'ROOT', root), + mock.patch.object(run_full_smoke, 'demo_environment', return_value=values), + mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run), + mock.patch.object(run_full_smoke, 'check_full'), + mock.patch.dict(os.environ)): + with self.assertRaisesRegex(AssertionError, 'minio-init'): + run_full_smoke.main() + self.assertFalse(any('up' in args for args in calls)) + receipt = json.loads((root / 'target/smoke-diagnostics/pulls.json').read_text()) + failed = receipt['pulls'][-1] + self.assertEqual('public.ecr.aws', failed['registry']) + self.assertEqual(model['services']['minio-init']['image'], failed['image']) + self.assertEqual(1, failed['result']['exit_code']) + self.assertIn('Rate exceeded', failed['result']['output']) + text = json.dumps(receipt) + self.assertNotIn('pull-secret', text) + self.assertNotIn('raw-config-fixture', text) + + def test_successful_pulls_use_same_images_without_building_or_starting(self): + module = self.diagnostics_module() + with tempfile.TemporaryDirectory() as directory: + evidence = module.Diagnostics(Path(directory), {}) + self.assertTrue(hasattr(evidence, 'pull'), 'explicit pull diagnostics not implemented') + calls = [] + model = {'services': {'postgres': {'image': 'postgres:16-alpine'}, + 'minio': {'image': 'qqq-all-minio:local', 'build': {'context': './infra/minio'}}, + 'mongo': {'image': 'mongo:7'}}} + def run(args, **kwargs): + calls.append(args) + output = json.dumps(model) if 'config' in args else 'pulled' + return subprocess.CompletedProcess(args, 0, output, '') + with mock.patch.object(module.subprocess, 'run', side_effect=run): + evidence.pull(['docker', 'compose', '-p', 'private'], Path(directory), {}) + pulls = [args for args in calls if 'pull' in args] + self.assertEqual(['mongo', 'postgres'], [args[-1] for args in pulls]) + for args in pulls: + self.assertEqual('1', args[args.index('--parallel') + 1]) + self.assertEqual('missing', args[args.index('--policy') + 1]) + self.assertFalse(any('up' in args or 'build' in args for args in calls)) + receipt = json.loads((Path(directory) / 'pulls.json').read_text()) + self.assertEqual(['mongo:7', 'postgres:16-alpine'], [r['image'] for r in receipt['pulls']]) + self.assertEqual(['docker.io', 'docker.io'], [r['registry'] for r in receipt['pulls']]) + def diagnostics_module(self): try: import smoke_diagnostics -- 2.54.0