Merge pull request #28 from QRun-IO/feature/GH-992-smoke-diagnostics

ci(GH-992): retain sanitized full smoke diagnostics

.github/workflows/smoke.yml

@@ -42,6 +42,17 @@ jobs:
           java-version: "21"
           cache: maven
       - name: Package full app for Docker image
-        run: mvn -B -ntp -pl qqq-all-app -am -DskipTests package
+        run: >-
+          mvn -B -ntp -pl qqq-all-app -am -DskipTests
+          dependency:build-classpath -DincludeGroupIds=com.kingsrook.qqq,com.kingsrook.qbits
+          -Dmdep.outputFile=target/smoke-classpath.txt package
       - name: Smoke full Compose over HTTP
         run: python3 scripts/run_full_smoke.py
+      - name: Retain sanitized full smoke diagnostics
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: full-smoke-diagnostics-${{ github.run_id }}-${{ github.run_attempt }}
+          path: target/smoke-diagnostics/*.json
+          if-no-files-found: warn
+          retention-days: 14

scripts/run_full_smoke.py

@@ -12,6 +12,7 @@ import urllib.parse
 from pathlib import Path
 
 from smoke import check_full
+from smoke_diagnostics import Diagnostics
 
 
 ROOT = Path(__file__).resolve().parent.parent
@@ -95,6 +96,7 @@ def main():
     environment = os.environ.copy()
     environment.update(values)
     os.environ.update(values)
+    diagnostics = Diagnostics(ROOT / "target/smoke-diagnostics", environment)
     with tempfile.TemporaryDirectory(prefix="qqq-all-full-smoke-") as directory:
         env_file = Path(directory) / "compose.env"
         env_file.write_text("".join(f"{key}={value}\n" for key, value in values.items()))
@@ -103,10 +105,12 @@ def main():
                    "-f", str(ROOT / "compose.yaml"), "-f", str(ROOT / "scripts/compose.smoke.yaml"),
                    "--profile", "full"]
         try:
-            started = subprocess.run(command + ["up", "-d", "--build", "--wait", "--wait-timeout", "360"],
-                                     cwd=ROOT, env=environment, stdout=subprocess.DEVNULL,
-                                     stderr=subprocess.DEVNULL, timeout=480, check=False)
-            if started.returncode:
+            diagnostics.jar_hashes(ROOT)
+            diagnostics.pull(command, ROOT, environment)
+            started = diagnostics.run(command + ["up", "-d", "--build", "--wait", "--wait-timeout", "360"],
+                                      ROOT, environment, timeout=480)
+            diagnostics.write("startup.json", started)
+            if started["exit_code"] != 0:
                 raise AssertionError("full Compose stack did not become healthy")
             prepare_sftp_import_directory(command, environment, values.get("SFTP_USER", "qqq"))
             check_full(
@@ -115,10 +119,22 @@ def main():
                 lambda index, query: query_opensearch(command, environment, index, query),
                 120,
             )
+        except (AssertionError, OSError, ValueError, subprocess.TimeoutExpired) as error:
+            diagnostics.write("failure.json", {"type": type(error).__name__,
+                                              "message": diagnostics.sanitize(str(error))})
+            raise AssertionError(diagnostics.sanitize(str(error))) from None
         finally:
-            subprocess.run(command + ["down", "--volumes", "--remove-orphans"],
-                           cwd=ROOT, env=environment, stdout=subprocess.DEVNULL,
-                           stderr=subprocess.DEVNULL, timeout=120, check=False)
+            try:
+                diagnostics.collect(command, ROOT, environment)
+            except Exception as error:
+                # Evidence failure must not suppress the smoke error or private-stack cleanup.
+                print(f"smoke diagnostics collection failed: {type(error).__name__}", file=sys.stderr)
+            finally:
+                cleanup = diagnostics.run(command + ["down", "--volumes", "--remove-orphans"],
+                                          ROOT, environment, timeout=120)
+                diagnostics.write("cleanup.json", cleanup)
+                if cleanup["exit_code"] != 0 and sys.exc_info()[0] is None:
+                    raise AssertionError("private Compose cleanup failed; see sanitized diagnostics")
 
 
 if __name__ == "__main__":

scripts/smoke_diagnostics.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""Small, sanitized receipts for the private full-smoke stack only."""
+
+import base64
+import hashlib
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import urllib.parse
+from pathlib import Path
+
+
+LIMIT = 65536
+CONTAINER_FORMAT = ('{"id":{{json .Id}},"service":{{json (index .Config.Labels '
+                    '"com.docker.compose.service")}},"image_id":{{json .Image}},'
+                    '"status":{{json .State.Status}},"exit_code":{{json .State.ExitCode}},'
+                    '"oom_killed":{{json .State.OOMKilled}},"error":{{json .State.Error}},'
+                    '"health":{{if index .State "Health"}}{{json (index .State "Health")}}{{else}}null{{end}}}')
+IMAGE_FORMAT = ('{"id":{{json .Id}},"repo_digests":{{json .RepoDigests}},'
+                '"architecture":{{json .Architecture}},"os":{{json .Os}}}')
+
+
+class Diagnostics:
+    def __init__(self, directory, environment):
+        self.directory = directory
+        self.secrets = set()
+        for key, value in environment.items():
+            if value and re.search(r'PASSWORD|PASSWD|SECRET|TOKEN|KEYFILE|ACCESS_KEY|PRIVATE_KEY|CREDENTIAL|AUTHORIZATION', key, re.I):
+                self.secrets.update((value, urllib.parse.quote(value, safe=''),
+                                     urllib.parse.quote_plus(value),
+                                     base64.b64encode(value.encode()).decode()))
+
+    def sanitize(self, text):
+        if isinstance(text, bytes):
+            text = text.decode('utf-8', errors='replace')
+        text = text or ''
+        text = re.sub(r'\x1b\[[0-?]*[ -/]*[@-~]', '', text)
+        for secret in sorted(self.secrets, key=len, reverse=True):
+            text = text.replace(secret, '[REDACTED]')
+        text = re.sub(r'(?i)(\b[\w.-]{0,80}(?:password|passwd|secret|token|authorization|credential|api[_-]?key|access[_-]?key)\b["\s]*[:=]\s*)("[^"]*"|\x27[^\x27]*\x27|[^\s,;]+)', r'\1[REDACTED]', text)
+        text = re.sub(r'(?i)\b(Bearer|Basic)\s+[^\s,;]+', r'\1 [REDACTED]', text)
+        text = re.sub(r'(?i)\b([a-z][a-z0-9+.-]*://)[^/\s@]+@', r'\1[REDACTED]@', text)
+        if len(text) > LIMIT:
+            text = '[...truncated...]\n' + text[-(LIMIT - 18):]
+        return text
+
+    def write(self, name, data):
+        # Redact strings before JSON encoding, so escaped secrets cannot escape filtering.
+        def clean(value):
+            if isinstance(value, str):
+                return self.sanitize(value)
+            if isinstance(value, dict):
+                return {key: clean(item) for key, item in value.items()}
+            if isinstance(value, list):
+                return [clean(item) for item in value]
+            return value
+        try:
+            self.directory.mkdir(parents=True, exist_ok=True)
+            path = self.directory / name
+            path.write_text(json.dumps(clean(data), indent=2) + '\n')
+            path.chmod(0o600)
+        except OSError as error:
+            print(f"smoke diagnostics write failed: {type(error).__name__}", file=sys.stderr)
+
+    def run(self, args, root, environment, timeout=15):
+        try:
+            result = subprocess.run(args, cwd=root, env=environment, capture_output=True,
+                                    text=True, timeout=timeout, check=False)
+            return {'exit_code': result.returncode,
+                    'output': self.sanitize((result.stdout or '') + (result.stderr or ''))}
+        except subprocess.TimeoutExpired as error:
+            return {'exit_code': None, 'error': 'TimeoutExpired',
+                    'output': self.sanitize(self.sanitize(error.stdout) + self.sanitize(error.stderr))}
+        except OSError as error:
+            return {'exit_code': None, 'error': type(error).__name__, 'output': ''}
+
+    def jar_hashes(self, root):
+        data = {'dependencies': []}
+        manifest = root / 'qqq-all-app/target/smoke-classpath.txt'
+        if manifest.is_file():
+            for name in sorted(set(manifest.read_text().strip().split(os.pathsep))):
+                path = Path(name)
+                match = re.search(r'/com/kingsrook/(qqq|qbits)/([^/]+)/([^/]+)/[^/]+\.jar$', path.as_posix())
+                if not match:
+                    continue
+                group, artifact, version = match.groups()
+                entry = {'coordinate': f'com.kingsrook.{group}:{artifact}:{version}', 'file': path.name}
+                if path.is_file():
+                    with path.open('rb') as stream:
+                        entry['sha256'] = hashlib.file_digest(stream, 'sha256').hexdigest()
+                else:
+                    entry['error'] = 'resolved JAR missing'
+                data['dependencies'].append(entry)
+        else:
+            data['error'] = 'resolved classpath missing; run the full-smoke Maven command'
+        app = root / 'qqq-all-app/target/qqq-all-app.jar'
+        if app.is_file():
+            with app.open('rb') as stream:
+                data['application_sha256'] = hashlib.file_digest(stream, 'sha256').hexdigest()
+        self.write('jars.json', data)
+
+    def pull(self, command, root, environment):
+        receipt = {'pulls': []}
+        try:
+            # Keep the uninterpolated model in memory only; retain no service environment.
+            model = subprocess.run(command + ['config', '--format', 'json', '--no-interpolate',
+                                              '--no-env-resolution'], cwd=root, env=environment,
+                                   capture_output=True, text=True, timeout=15, check=False)
+            if model.returncode:
+                raise ValueError('Compose image selection failed')
+            services = json.loads(model.stdout)['services']
+            if not isinstance(services, dict) or not 1 <= len(services) <= 32:
+                raise ValueError('unexpected service inventory')
+            selected = [(name, service['image']) for name, service in sorted(services.items())
+                        if 'build' not in service]
+            if any(not re.fullmatch(r'[a-zA-Z0-9][a-zA-Z0-9_.-]*', name)
+                   or not isinstance(image, str) or '${' in image for name, image in selected):
+                raise ValueError('unresolved service image')
+        except (KeyError, TypeError, ValueError, OSError, subprocess.TimeoutExpired) as error:
+            receipt['selection_error'] = type(error).__name__
+            self.write('pulls.json', receipt)
+            raise AssertionError('could not select Compose service images; see pulls.json') from None
+        deadline = time.monotonic() + 480
+        for service, image in selected:
+            first = image.split('/')[0]
+            registry = first if '/' in image and ('.' in first or ':' in first or first == 'localhost') else 'docker.io'
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                result = {'exit_code': None, 'error': 'pull-stage budget exhausted', 'output': ''}
+            else:
+                result = self.run(command + ['--parallel', '1', '--progress', 'plain', 'pull',
+                                             '--ignore-buildable', '--policy', 'missing', service],
+                                  root, environment, timeout=min(120, remaining))
+            receipt['pulls'].append({'service': service, 'image': image,
+                                     'registry': registry, 'result': result})
+            self.write('pulls.json', receipt)
+            if result['exit_code'] != 0:
+                raise AssertionError(f'Compose pull failed for {service} ({image}); see pulls.json')
+
+    def collect(self, command, root, environment):
+        deadline = time.monotonic() + 60
+        def run(args):
+            remaining = deadline - time.monotonic()
+            if remaining <= 0:
+                return {'exit_code': None, 'error': 'collection budget exhausted', 'output': ''}
+            return self.run(args, root, environment, timeout=min(15, remaining))
+        inventory = run(command + ['ps', '-a', '-q'])
+        ids = [line for line in inventory['output'].splitlines()
+               if re.fullmatch(r'[a-f0-9]{12,64}', line)][:32]
+        services, images = [], {}
+        for container in ids:
+            result = run(['docker', 'inspect', '--format', CONTAINER_FORMAT, container])
+            try:
+                raw = json.loads(result['output'])
+                state = {key: raw.get(key) for key in ('id', 'service', 'image_id', 'status',
+                                                       'exit_code', 'oom_killed', 'error')}
+                health = raw.get('health') or {}
+                state['health'] = {'status': health.get('Status'),
+                                  'checks': [{key: check.get(key) for key in ('Start', 'End', 'ExitCode', 'Output')}
+                                             for check in health.get('Log', [])[-5:]]}
+                services.append(state)
+                image = state['image_id']
+                if isinstance(image, str) and re.fullmatch(r'sha256:[a-f0-9]{64}', image) and image not in images:
+                    found = run(['docker', 'image', 'inspect', '--format', IMAGE_FORMAT, image])
+                    try:
+                        details = json.loads(found['output'])
+                        images[image] = {key: details.get(key) for key in ('id', 'repo_digests', 'architecture', 'os')}
+                    except (ValueError, TypeError, AttributeError):
+                        images[image] = {'error': 'image identity unavailable', 'command': found}
+            except (ValueError, TypeError, AttributeError):
+                services.append({'id': container, 'error': 'container state unavailable', 'command': result})
+            self.write(f'container-{container[:12]}.json',
+                       run(['docker', 'logs', '--timestamps', '--tail', '100', container]))
+        self.write('services.json', {'inventory': inventory, 'services': services, 'images': images})

scripts/test_smoke.py

@@ -192,6 +192,7 @@ class SmokeTest(unittest.TestCase):
             with (mock.patch.object(run_full_smoke, "ROOT", root),
                   mock.patch.object(run_full_smoke, "demo_environment", return_value=values),
                   mock.patch.object(run_full_smoke, "published_port", return_value=20003),
+                  mock.patch.object(run_full_smoke.Diagnostics, "pull"),
                   mock.patch.object(run_full_smoke.subprocess, "run",
                                     return_value=subprocess.CompletedProcess([], 0)),
                   mock.patch.object(run_full_smoke, "check_full", side_effect=verify_credentials) as check,

scripts/test_smoke_diagnostics.py

@@ -0,0 +1,255 @@
+#!/usr/bin/env python3
+"""Failure evidence must survive cleanup without leaking runtime credentials."""
+
+import hashlib
+import json
+import os
+import subprocess
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import run_full_smoke
+
+
+class DiagnosticsTest(unittest.TestCase):
+    def test_startup_failure_retains_redacted_evidence_before_cleanup(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / 'compose.yaml').touch()
+            values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002',
+                      'POSTGRES_PASSWORD': 'private-password-123'}
+            cleaned = []
+
+            def run(args, **kwargs):
+                if 'config' in args:
+                    return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '')
+                if 'up' in args:
+                    return subprocess.CompletedProcess(args, 1, 'postgres unhealthy private-password-123', '')
+                if 'down' in args:
+                    receipt = root / 'target/smoke-diagnostics/startup.json'
+                    self.assertTrue(receipt.is_file(), 'diagnostics must exist before teardown')
+                    cleaned.append(True)
+                return subprocess.CompletedProcess(args, 0, '', '')
+
+            with (mock.patch.object(run_full_smoke, 'ROOT', root),
+                  mock.patch.object(run_full_smoke, 'demo_environment', return_value=values),
+                  mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run),
+                  mock.patch.dict(os.environ)):
+                with self.assertRaisesRegex(AssertionError, 'did not become healthy'):
+                    run_full_smoke.main()
+            self.assertTrue(cleaned)
+            output = (root / 'target/smoke-diagnostics/startup.json').read_text()
+            self.assertIn('postgres unhealthy', output)
+            self.assertNotIn('private-password-123', output)
+            self.assertEqual(1, json.loads(output)['exit_code'])
+
+    def test_http_failure_stays_sanitized_when_collection_and_cleanup_fail(self):
+        self.diagnostics_module()
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / 'compose.yaml').touch()
+            values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002',
+                      'DB_PASSWORD': 'hidden-fixture'}
+            def run(args, **kwargs):
+                if 'config' in args:
+                    return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '')
+                if 'down' in args:
+                    raise subprocess.TimeoutExpired(args, 120, output='hidden-fixture')
+                return subprocess.CompletedProcess(args, 0, '', '')
+            with (mock.patch.object(run_full_smoke, 'ROOT', root),
+                  mock.patch.object(run_full_smoke, 'demo_environment', return_value=values),
+                  mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run),
+                  mock.patch.object(run_full_smoke.Diagnostics, 'collect', side_effect=ValueError('hidden-fixture')),
+                  mock.patch.object(run_full_smoke, 'check_full', side_effect=AssertionError('HTTP failed hidden-fixture')),
+                  mock.patch.dict(os.environ)):
+                with self.assertRaisesRegex(AssertionError, r'HTTP failed \[REDACTED\]'):
+                    run_full_smoke.main()
+            text = ''.join(f.read_text() for f in (root / 'target/smoke-diagnostics').glob('*.json'))
+            self.assertNotIn('hidden-fixture', text)
+            self.assertIn('TimeoutExpired', text)
+
+    def test_startup_timeout_keeps_partial_sanitized_output_and_tears_down(self):
+        self.diagnostics_module()
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / 'compose.yaml').touch()
+            values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002',
+                      'DB_PASSWORD': 'hidden-fixture'}
+            cleanup = []
+            def run(args, **kwargs):
+                if 'config' in args:
+                    return subprocess.CompletedProcess(args, 0, json.dumps({'services': {'db': {'image': 'postgres:16-alpine'}}}), '')
+                if 'up' in args:
+                    raise subprocess.TimeoutExpired(args, 480, output=b'postgres unhealthy hidden-fixture')
+                if 'down' in args:
+                    cleanup.append(True)
+                return subprocess.CompletedProcess(args, 0, '', '')
+            with (mock.patch.object(run_full_smoke, 'ROOT', root),
+                  mock.patch.object(run_full_smoke, 'demo_environment', return_value=values),
+                  mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run),
+                  mock.patch.dict(os.environ)):
+                with self.assertRaisesRegex(AssertionError, 'did not become healthy'):
+                    run_full_smoke.main()
+            self.assertTrue(cleanup)
+            receipt = json.loads((root / 'target/smoke-diagnostics/startup.json').read_text())
+            self.assertEqual('TimeoutExpired', receipt['error'])
+            self.assertIn('postgres unhealthy', receipt['output'])
+            self.assertNotIn('hidden-fixture', receipt['output'])
+
+    def test_pull_failure_names_service_image_registry_and_stops_before_up(self):
+        module = self.diagnostics_module()
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / 'compose.yaml').touch()
+            values = {'QQQ_ALL_PORT': '20001', 'KEYCLOAK_PORT': '20002',
+                      'DB_PASSWORD': 'pull-secret'}
+            calls = []
+            model = {'services': {
+                'app-full': {'image': 'qqq-all:local', 'build': {'context': '.'}},
+                'minio-init': {'image': 'public.ecr.aws/aws-cli/aws-cli:2.27.41',
+                               'environment': {'DO_NOT_RETAIN': 'raw-config-fixture'}},
+                'sftp': {'image': 'atmoz/sftp:alpine'}}}
+            def run(args, **kwargs):
+                calls.append(args)
+                if 'config' in args:
+                    self.assertIn('--no-interpolate', args)
+                    self.assertIn('--no-env-resolution', args)
+                    return subprocess.CompletedProcess(args, 0, json.dumps(model), '')
+                if 'pull' in args:
+                    self.assertEqual('minio-init', args[-1])
+                    self.assertNotIn('--ignore-pull-failures', args)
+                    self.assertNotIn('--include-deps', args)
+                    return subprocess.CompletedProcess(args, 1, '', 'toomanyrequests: Rate exceeded pull-secret')
+                return subprocess.CompletedProcess(args, 0, '', '')
+            with (mock.patch.object(run_full_smoke, 'ROOT', root),
+                  mock.patch.object(run_full_smoke, 'demo_environment', return_value=values),
+                  mock.patch.object(run_full_smoke.subprocess, 'run', side_effect=run),
+                  mock.patch.object(run_full_smoke, 'check_full'),
+                  mock.patch.dict(os.environ)):
+                with self.assertRaisesRegex(AssertionError, 'minio-init'):
+                    run_full_smoke.main()
+            self.assertFalse(any('up' in args for args in calls))
+            receipt = json.loads((root / 'target/smoke-diagnostics/pulls.json').read_text())
+            failed = receipt['pulls'][-1]
+            self.assertEqual('public.ecr.aws', failed['registry'])
+            self.assertEqual(model['services']['minio-init']['image'], failed['image'])
+            self.assertEqual(1, failed['result']['exit_code'])
+            self.assertIn('Rate exceeded', failed['result']['output'])
+            text = json.dumps(receipt)
+            self.assertNotIn('pull-secret', text)
+            self.assertNotIn('raw-config-fixture', text)
+
+    def test_successful_pulls_use_same_images_without_building_or_starting(self):
+        module = self.diagnostics_module()
+        with tempfile.TemporaryDirectory() as directory:
+            evidence = module.Diagnostics(Path(directory), {})
+            self.assertTrue(hasattr(evidence, 'pull'), 'explicit pull diagnostics not implemented')
+            calls = []
+            model = {'services': {'postgres': {'image': 'postgres:16-alpine'},
+                                  'minio': {'image': 'qqq-all-minio:local', 'build': {'context': './infra/minio'}},
+                                  'mongo': {'image': 'mongo:7'}}}
+            def run(args, **kwargs):
+                calls.append(args)
+                output = json.dumps(model) if 'config' in args else 'pulled'
+                return subprocess.CompletedProcess(args, 0, output, '')
+            with mock.patch.object(module.subprocess, 'run', side_effect=run):
+                evidence.pull(['docker', 'compose', '-p', 'private'], Path(directory), {})
+            pulls = [args for args in calls if 'pull' in args]
+            self.assertEqual(['mongo', 'postgres'], [args[-1] for args in pulls])
+            for args in pulls:
+                self.assertEqual('1', args[args.index('--parallel') + 1])
+                self.assertEqual('missing', args[args.index('--policy') + 1])
+            self.assertFalse(any('up' in args or 'build' in args for args in calls))
+            receipt = json.loads((Path(directory) / 'pulls.json').read_text())
+            self.assertEqual(['mongo:7', 'postgres:16-alpine'], [r['image'] for r in receipt['pulls']])
+            self.assertEqual(['docker.io', 'docker.io'], [r['registry'] for r in receipt['pulls']])
+
+    def diagnostics_module(self):
+        try:
+            import smoke_diagnostics
+        except ImportError:
+            self.fail('sanitized diagnostics collector is not implemented')
+        return smoke_diagnostics
+
+    def test_redacts_secret_forms_and_bounds_output_after_redaction(self):
+        module = self.diagnostics_module()
+        with tempfile.TemporaryDirectory() as directory:
+            evidence = module.Diagnostics(Path(directory), {'DB_PASSWORD': 'a+/secret',
+                                                           'GITHUB_TOKEN': 'token-1234'})
+            output = evidence.sanitize('x' * 90000 + '\npassword="unknown-value" '
+                                       'postgres://u:uri-secret@db/ Bearer bearer-secret '
+                                       'a%2B%2Fsecret a+/secret token-1234 access_token=generated-token client_secret=generated-secret')
+            for secret in ['unknown-value', 'uri-secret', 'bearer-secret',
+                           'a%2B%2Fsecret', 'a+/secret', 'token-1234', 'generated-token', 'generated-secret']:
+                self.assertFalse(secret in output, "credential remained in sanitized output")
+            self.assertLessEqual(len(output), 65536)
+            self.assertIn('[REDACTED]', output)
+
+    def test_hashes_only_resolved_first_party_jars_not_other_cached_versions(self):
+        module = self.diagnostics_module()
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            jar = root / 'repository/com/kingsrook/qqq/qqq-esb/4.1.0-SNAPSHOT/qqq-esb.jar'
+            jar.parent.mkdir(parents=True)
+            jar.write_bytes(b'actual resolved bytes')
+            (jar.parent / 'unused.jar').write_bytes(b'unused')
+            manifest = root / 'qqq-all-app/target/smoke-classpath.txt'
+            manifest.parent.mkdir(parents=True)
+            manifest.write_text(str(jar))
+            evidence = module.Diagnostics(root / 'target/smoke-diagnostics', {})
+            evidence.jar_hashes(root)
+            data = json.loads((evidence.directory / 'jars.json').read_text())
+            self.assertEqual([{'coordinate': 'com.kingsrook.qqq:qqq-esb:4.1.0-SNAPSHOT',
+                               'file': 'qqq-esb.jar',
+                               'sha256': hashlib.sha256(b'actual resolved bytes').hexdigest()}], data['dependencies'])
+            self.assertNotIn(str(root), json.dumps(data))
+
+    def test_container_collection_is_selected_redacted_and_survives_command_failure(self):
+        module = self.diagnostics_module()
+        with tempfile.TemporaryDirectory() as directory:
+            evidence = module.Diagnostics(Path(directory), {'DB_PASSWORD': 'secret-fixture'})
+            container = 'a' * 64
+            completed = 'd' * 64
+            image = 'sha256:' + 'b' * 64
+            commands = []
+            def run(args, **kwargs):
+                commands.append(args)
+                if 'ps' in args:
+                    return subprocess.CompletedProcess(args, 0, container + '\n' + completed + '\n', '')
+                if 'inspect' in args and 'image' not in args:
+                    self.assertIn('--format', args)
+                    if args[-1] == completed:
+                        return subprocess.CompletedProcess(args, 0, json.dumps({
+                            'id': completed, 'service': 'mongo-init', 'image_id': image,
+                            'status': 'exited', 'exit_code': 0, 'oom_killed': False,
+                            'error': '', 'health': None}), '')
+                    return subprocess.CompletedProcess(args, 0, json.dumps({
+                        'id': container, 'service': 'postgres', 'image_id': image,
+                        'status': 'exited', 'exit_code': 1, 'oom_killed': False,
+                        'error': '', 'health': {'Status': 'unhealthy', 'Log': [
+                            {'ExitCode': 1, 'Output': 'failed secret-fixture'}]}}), '')
+                if 'image' in args:
+                    return subprocess.CompletedProcess(args, 0, json.dumps({'id': image,
+                        'repo_digests': ['postgres@sha256:' + 'c' * 64],
+                        'architecture': 'amd64', 'os': 'linux'}), '')
+                if 'logs' in args:
+                    raise subprocess.TimeoutExpired(args, 15, output='secret-fixture')
+                self.fail('unexpected command')
+            with mock.patch.object(module.subprocess, 'run', side_effect=run):
+                evidence.collect(['docker', 'compose', '-p', 'private'], Path(directory), {})
+            text = '\n'.join(f.read_text() for f in Path(directory).glob('*.json'))
+            self.assertNotIn('secret-fixture', text)
+            self.assertIn('unhealthy', text)
+            self.assertIn('amd64', text)
+            states = json.loads((Path(directory) / 'services.json').read_text())['services']
+            self.assertEqual('mongo-init', states[1]['service'])
+            self.assertEqual(0, states[1]['exit_code'])
+            self.assertIsNone(states[1]['health']['status'])
+            self.assertIn('TimeoutExpired', text)
+            self.assertFalse(any('config' in c or '--volumes' in c for c in commands))
+
+
+if __name__ == '__main__':
+    unittest.main()