diff --git a/.gitignore b/.gitignore index 60d1228..1e13f27 100644 --- a/.gitignore +++ b/.gitignore @@ -27,5 +27,3 @@ SOURCE-MANIFEST.tsv SOURCE-MANIFEST.sha256 *.private.* *.log -# Prepared caller retained as an inert documentation reference while CI/CD is on hold. -.gitea/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..98721a9 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,23 @@ +# Changelog + +All notable changes are documented here, following [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) and semantic versioning. Record Files is pre-1.0: a `0.1.0-RC.N` release candidate is a published, immutable artifact, not a production 1.0 claim (see [docs/RELEASE-STATUS.md](docs/RELEASE-STATUS.md)). + +## [Unreleased] + +Planned as **0.1.0-RC.1**, the first artifact published from this repository. It is published by pushing `develop` to `release/0.1`; the shared `KofTwentyTwo/ci-workflows` Maven workflow computes the `0.1.0-RC.N` version, deploys to the Gitea Maven registry (`https://git.kof22.com/api/packages/QRun-IO/maven`) and tags `v0.1.0-RC.N`. Nothing is published from `develop` except the `0.1.0-SNAPSHOT`. + +### Added + +- First published release candidate of the standalone QBit extracted from Website-Backend `b6fb2382`: retained files, ordered collections, review vocabulary, append-only audit, resumable upload sessions, tracked downloads, archive-and-delete, verified print observations, and the `tests` classifier with the public integration fixtures. +- `CHANGELOG.md`, and a release-status document that records the publication path, the dependency cohort and the Forge development pin this release replaces. +- Test-only PostgreSQL fixture diagnostics from the GG Forge qualification (`record-files-postgres-diagnostics`, `record-files-linux-fixture`): `POSTGRES_FIXTURE_HEALTH` container observation after each PostgreSQL intake case, and `FixtureEndpoint`, which resolves the disposable container's reachable host and published port (honouring `TESTCONTAINERS_HOST_OVERRIDE`, with the sibling bridge address for a JVM inside a build container) instead of hard-coding `127.0.0.1`. This is what lets the suite run on the shared `container-build` runners. + +### Changed + +- Archive capture (`ArchiveSecurity.digest`) now tolerates host table customizers that run only on insert, update or delete (`PRE_/POST_INSERT_RECORD`, `PRE_/POST_UPDATE_RECORD`, `PRE_/POST_DELETE_RECORD`), since they cannot change what a reader may see. Query, get and personalization customizers still refuse the capture with "Archive capture does not support this table's custom visibility policy." This is the source form of the change that GG Forge carried as the `-archive1` suffix of its development pin `0.1.0-forge.162af9a7b8cd-archive1`; it was recovered from that artifact's bytecode and is covered by `ArchiveTest.mutationCustomizersAreToleratedWhileVisibilityCustomizersRefuseCapture`. +- `.gitea/` is no longer ignored, so the shared CI caller (`.gitea/workflows/build.yaml`, PR 2) is tracked normally; the retired platform-ci caller in `docs/release/qrun-release.reference.yaml` is marked superseded. + +### Known limits carried into the release candidate + +- The build needs a Collaboration artifact that carries the attachment lifecycle, upload journal and download SPI (Collaboration PR 59). Public Collaboration `0.1.0-RC.6` does not have it, and the `qbit-collaboration.version` property still names `0.1.0-SNAPSHOT`; pin the published Collaboration release candidate before pushing `release/0.1`. +- Everything in [docs/KNOWN-ISSUES.md](docs/KNOWN-ISSUES.md) still applies: no malware scanning, no WORM proof, PostgreSQL script unqualified for production, no complete public-cohort consumer installation yet. diff --git a/README.md b/README.md index c409df1..1202be8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # QBit Record Files -> **Paused, unreleased work in progress — 2026-11-04.** Implementation has stopped. This is a source checkpoint, not a production-ready release or a complete installable package set. +> **First release candidate in preparation — 2026-11-06.** `0.1.0-RC.1` will be the first artifact published from this repository (Gitea Maven registry, via the shared ci-workflows Maven workflow). It is a release candidate, not a production 1.0, and it is not published yet. Start with [current status and build prerequisites](docs/RELEASE-STATUS.md) and [known issues](docs/KNOWN-ISSUES.md). These documents supersede older release-readiness statements below. @@ -93,3 +93,10 @@ Requires JDK 21 and a `qbit-collaboration` release that contains the attachment Published POMs are self-contained: the estate's `flatten-maven-plugin` (`ossrh` mode) resolves `${revision}` and the version properties at build time, so a consumer needs no `-Drevision` override. The version comes from `` in `pom.xml`. The inherited `qbit-build-parent` POM still warns until its owner publishes a literal version. Release candidate, not GA. + +Artifacts are published to the Gitea Maven registry, `https://git.kof22.com/api/packages/QRun-IO/maven`, by the shared +`KofTwentyTwo/ci-workflows` Maven workflow (`.gitea/workflows/build.yaml`): `develop` deploys `0.1.0-SNAPSHOT`, a push to +`release/0.1` publishes the next `0.1.0-RC.N` and tags it, and a `v0.1.0` tag publishes the stable release. The registry +requires sign-in for reads; configure a `` for it in your private Maven settings. See +[docs/RELEASE-STATUS.md](docs/RELEASE-STATUS.md) for the release steps and [CHANGELOG.md](CHANGELOG.md) for what each +version contains. diff --git a/docs/KNOWN-ISSUES.md b/docs/KNOWN-ISSUES.md index 3651256..1abd718 100644 --- a/docs/KNOWN-ISSUES.md +++ b/docs/KNOWN-ISSUES.md @@ -1,9 +1,9 @@ # Known issues and unfinished work -Updated: 2026-11-04. These are unresolved items, not a plan currently being executed. +Updated: 2026-11-06. These are unresolved items carried into the first release candidate, not a plan currently being executed. - Public Collaboration 0.1.0-RC.6 does not provide the attachment lifecycle, upload journal, recovery and download APIs this module requires. -- The corresponding canonical Collaboration integration is preserved on a separate work-in-progress branch. It requires confirmed rollback callbacks in QQQ Core/RDBMS; public QQQ 4.1.0-RC.21 is insufficient for that dependency chain. -- Current snapshot dependency versions are placeholders for source integration, not published installation coordinates. No clean consumer installation of the complete public cohort has been demonstrated. +- The corresponding canonical Collaboration integration is Collaboration PR 59 (`kof22/collab-v2`), not yet published. It requires confirmed rollback callbacks in QQQ Core/RDBMS; public QQQ 4.1.0-RC.21 is insufficient for that dependency chain. +- `qbit-collaboration.version` is still `0.1.0-SNAPSHOT`, a placeholder for source integration. The release candidate must pin a published Collaboration RC before `release/0.1` is pushed (see RELEASE-STATUS.md). No clean consumer installation of the complete public cohort has been demonstrated. - Full Record Files verification, database integration, retained-file recovery and final standard-UI acceptance remain incomplete for this reconciled source. - Permanent access depends on the host's durable object storage, backups, retention and permission configuration. A source-level interface or unit test alone does not prove operational permanence. diff --git a/docs/RELEASE-STATUS.md b/docs/RELEASE-STATUS.md index c798e0d..ab41c2c 100644 --- a/docs/RELEASE-STATUS.md +++ b/docs/RELEASE-STATUS.md @@ -1,6 +1,6 @@ # Record Files — current status -Updated: 2026-11-04. **Work stopped at the user's request. Unreleased work in progress; not a production 1.0 release.** +Updated: 2026-11-06. **Preparing the first published release candidate, `0.1.0-RC.1`.** Not a production 1.0; see [KNOWN-ISSUES.md](KNOWN-ISSUES.md) and [TODO.md](TODO.md) for the gates that remain. ## Purpose @@ -10,18 +10,51 @@ Retained uploads, scanned files and photo collections associated with QQQ record The root Maven module contains the QBit and its integration-test fixture classifier. Byte storage and attachment intake use the existing Collaboration attachment contracts. Generation and printing integrations live in their owning bridge modules, not here. -## Building and installation +## Building -Use Java 21 and the included Maven wrapper. After compatible Core and Collaboration artifacts are available, run `./mvnw verify`. Integration tests require their Docker/Testcontainers services. Do not expect the currently published older dependencies to compile this branch. +Use Java 21 and the included Maven wrapper: `./mvnw verify`. The PostgreSQL and MySQL integration suites are opt-in: they use Testcontainers (a Docker daemon is required) and run only with `-DpostgresRecordFilesTests=true` and `-DmysqlRecordFilesTests=true`; otherwise they are reported as skipped, so a plain `verify` is not provider evidence. The build needs a Collaboration artifact that carries the attachment lifecycle, upload journal and download SPI (`AttachmentLifecyclePolicy`, `AttachmentDownload`, `AttachmentContentIntegrityException`, `uploadjournal.UploadRecoveryNavigation`, `security.ParentRecordAccess`). Public Collaboration `0.1.0-RC.6` does not have them; that work is Collaboration [PR 59](https://git.kof22.com/QRun-IO/qbit-collaboration/pulls/59). Until it is published, build against a locally installed snapshot of that branch and override the property, for example `./mvnw -Dqbit-collaboration.version= verify`. + +## Version and publication path + +Versioning follows the shared [KofTwentyTwo/ci-workflows](https://git.kof22.com/KofTwentyTwo/ci-workflows) Maven workflow, called from `.gitea/workflows/build.yaml` (PR 2). The version lives in the `` property of `pom.xml` and stays `0.1.0-SNAPSHOT` on `develop`; the workflow computes published versions and commits the bump back. + +| Ref | Version | Action | +| --- | --- | --- | +| feature branches, `main` | unchanged | `mvn verify` only | +| `develop` | `0.1.0-SNAPSHOT` | test, deploy snapshot | +| `release/0.1` | `0.1.0-RC.N` | test, deploy, commit bump, tag `v0.1.0-RC.N` | +| tag `v0.1.0` | `0.1.0` | test, deploy, commit to `main` | + +Packages go to the Gitea Maven registry, `https://git.kof22.com/api/packages/QRun-IO/maven` (sign-in required for reads as well as writes). Nothing is published to Maven Central and nothing is GPG-signed. GitHub is a shadow/archive; never publish from it. + +### First release candidate, step by step + +1. Merge Collaboration PR 59 and publish its release candidate (`release/0.1` there), so the registry holds a Collaboration RC with the SPI above. +2. Merge PR 2 (shared CI caller) and this release-preparation PR into `develop`; confirm the `develop` push run passes and deploys `0.1.0-SNAPSHOT`. +3. Merge PR 1 (`quarantineCapturedFiles`) into `develop`; it has no source conflict with this branch and only adds to `RecordFilesQBitConfig` and `RetainedFileService`. +4. On `develop`, set `qbit-collaboration.version` to the published Collaboration RC (a release candidate must not depend on a `-SNAPSHOT`), then `git push origin develop:release/0.1`. The push run publishes `0.1.0-RC.1` and tags `v0.1.0-RC.1`. +5. Resolve `com.kingsrook.qbits:qbit-record-files:0.1.0-RC.1` (and its `tests` classifier if the fixture is wanted) from the registry in a fresh consumer cache and record the JAR and POM SHA-256 here. A passing build alone is not proof of publication. + +The stable `0.1.0` comes later from a `v0.1.0` tag on `main`; it is not part of this candidate. + +## What this candidate contains beyond the paused checkpoint (`162af9a`) + +- The archive-capture customizer guard change that GG Forge ran as development pin `0.1.0-forge.162af9a7b8cd-archive1` (mutation-only host customizers no longer refuse a capture; see CHANGELOG). +- The two test-only PostgreSQL fixture patches from the Forge qualification (`FixtureEndpoint`, fixture health reporting), unchanged from `scripts/local/patches/record-files-*.patch` in the Forge handoff. +- Release documentation. No behaviour other than the archive guard changed. + +### Relationship to the GG Forge development pin + +Forge's pin `0.1.0-forge.162af9a7b8cd-archive1` was built from the same source as this candidate, but against development coordinates: `qqq-backend-core 4.1.0-forge.33464d37e815-echo1`, `qqq-backend-module-rdbms 4.1.0-forge.33464d37e815` and `qbit-collaboration 0.1.0-forge.8f348daf324d-files1`. `0.1.0-RC.1` is a drop-in replacement for the Record Files code itself (same classes, plus the `quarantineCapturedFiles` setting once PR 1 is merged), but its POM pins the public QQQ `4.1.0-RC.21` tuple and the published Collaboration RC, so the host must move the whole cohort together, not only this coordinate. ## Evidence retained -The extracted source passed Checkstyle and flattened-POM checks. Compilation against public Collaboration 0.1.0-RC.6 exposed missing attachment lifecycle APIs. End-to-end verification of this standalone package has not completed. The earlier scanner integration proof used a private dependency cohort and does not constitute a standalone public release. +- Local `./mvnw -Dqbit-collaboration.version=0.1.0-rf53bd8f00-SNAPSHOT verify` on this branch (Collaboration commit `53bd8f00`, the commit PR 1 was verified against; Java 21): 284 tests, 0 failures, 0 errors, 58 skipped (the opt-in database suites), Checkstyle clean, BUILD SUCCESS. +- Opt-in database suites with Docker 29.9.2 (`-DpostgresRecordFilesTests=true -DmysqlRecordFilesTests=true`): 64 tests, 1 failure, 1 error. MySQL 15/15, PostgresTrackedDownload 3/3, PostgresRetainedUploadRecovery 5/5 and FixtureEndpoint 5/5 pass; `POSTGRES_FIXTURE_HEALTH` reports `running=true oom=false restarting=false endpointMatches=true` after all 22 intake cases. The two failures pre-date this branch: `PostgresIntakeAndRetentionTest.unmanagedTablesKeepOrdinaryAttachmentBehavior` dies in Collaboration's `AttachmentsService.storeUpload` with `NoSuchMethodError QBackendTransaction.supportsRollbackCallbacks()` (the documented QQQ core after-rollback-callback gap: public 4.1.0-RC.21 lacks it, Forge builds against `4.1.0-forge.33464d37e815-echo1`), and `PostgresRecordFilesTest.refusesLabelsAndPhysicalDrift` fails its `isDurableBackend` re-check after `ALTER COLUMN at TYPE TIMESTAMPTZ` (line 258), a physical-drift probe untouched by this change. +- The earlier scanner integration proof used a private dependency cohort and does not constitute a standalone public release. ## Source and release state -The starting source was extracted from Website-Backend commit b6fb23825a87412397dec703b9617148ae7eb601, then adapted for standalone packaging. This repository preserves the resulting source and existing licenses. It does not declare a published artifact cohort, completed deployment or complete acceptance. +The starting source was extracted from Website-Backend commit b6fb23825a87412397dec703b9617148ae7eb601, then adapted for standalone packaging. This repository preserves the resulting source and existing licenses. The outstanding dependency order is Core, Collaboration, Scripting/Record Files, then the integrated Generation bridges and matching UI. -See [known issues](KNOWN-ISSUES.md) and the README for the existing API and integration documentation. No further implementation, tests, releases or deployment should be started without explicit user instruction. The outstanding dependency order is Core, Collaboration, Scripting/Record Files, then the integrated Generation bridges and matching UI. - -The prepared release caller is saved as an inactive reference in docs/release/qrun-release.reference.yaml. No new workflow is being activated by this checkpoint; shared CI/CD design and enrollment remain unfinished. +The old platform-ci caller is kept, inactive, as `docs/release/qrun-release.reference.yaml`; the shared ci-workflows caller supersedes it. diff --git a/docs/release/qrun-release.reference.yaml b/docs/release/qrun-release.reference.yaml index a063d1a..9ddeb3b 100644 --- a/docs/release/qrun-release.reference.yaml +++ b/docs/release/qrun-release.reference.yaml @@ -1,3 +1,5 @@ +# SUPERSEDED, inactive reference. The repository now publishes through the shared KofTwentyTwo/ci-workflows Maven workflow +# (.gitea/workflows/build.yaml, PR 2); see docs/RELEASE-STATUS.md. Kept only as a record of the retired platform-ci caller. # Standard QRun release caller (v2, thin contract): every push and manual dispatch delegates to the protected qrun-release-v1 channel. Do not edit per repository. name: qrun-release on: diff --git a/src/main/java/com/kingsrook/qbits/recordfiles/archive/ArchiveSecurity.java b/src/main/java/com/kingsrook/qbits/recordfiles/archive/ArchiveSecurity.java index c2347a8..135cfcb 100644 --- a/src/main/java/com/kingsrook/qbits/recordfiles/archive/ArchiveSecurity.java +++ b/src/main/java/com/kingsrook/qbits/recordfiles/archive/ArchiveSecurity.java @@ -150,6 +150,18 @@ public final class ArchiveSecurity } } + /******************************************************************************* + ** Host customizers that only run on insert, update or delete cannot change what a + ** reader may see, so an archive capture tolerates them. Every other customizer role + ** (query, get, personalization) can shape visibility and is still refused. + *******************************************************************************/ + private static final Set MUTATION_ROLES = Set.of( + TableCustomizers.PRE_INSERT_RECORD.getRole(), TableCustomizers.POST_INSERT_RECORD.getRole(), + TableCustomizers.PRE_UPDATE_RECORD.getRole(), TableCustomizers.POST_UPDATE_RECORD.getRole(), + TableCustomizers.PRE_DELETE_RECORD.getRole(), TableCustomizers.POST_DELETE_RECORD.getRole()); + + + /******************************************************************************* ** Fingerprint exact native lock topology/types/key definitions; permission ** grants are intentionally not captured and are always evaluated currently. @@ -157,7 +169,7 @@ public final class ArchiveSecurity private static String digest(QTableMetaData table) throws QException { if(table == null || table.getPrimaryKeyField() == null || table.getCustomizers() != null - && table.getCustomizers().keySet().stream().anyMatch(role -> !TableCustomizers.PRE_DELETE_RECORD.getRole().equals(role)) + && table.getCustomizers().keySet().stream().anyMatch(role -> !MUTATION_ROLES.contains(role)) || QContext.getQInstance().getSupplementalCustomizer(com.kingsrook.qqq.backend.core.actions.metadata.personalization.TableMetaDataPersonalizerInterface.CUSTOMIZER_TYPE) != null) { throw (new QException("Archive capture does not support this table's custom visibility policy.")); diff --git a/src/test/java/com/kingsrook/qbits/recordfiles/ArchiveTest.java b/src/test/java/com/kingsrook/qbits/recordfiles/ArchiveTest.java index f4083a3..9a69fe4 100644 --- a/src/test/java/com/kingsrook/qbits/recordfiles/ArchiveTest.java +++ b/src/test/java/com/kingsrook/qbits/recordfiles/ArchiveTest.java @@ -237,6 +237,46 @@ public class ArchiveTest extends RecordFilesFixture } } + /******************************************************************************* + ** Host insert/update/delete customizers cannot change reader visibility, so a capture + ** tolerates them; a query-shaping customizer still refuses the capture before any change. + *******************************************************************************/ + @Test + void mutationCustomizersAreToleratedWhileVisibilityCustomizersRefuseCapture() throws Exception + { + var file = availableFile("archive-customizers", 1, "bol.txt", "evidence"); + var table = instance.getTable("document"); + table.withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.POST_QUERY_RECORD, + new com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference(NoOpCustomizer.class)); + assertThatThrownBy(() -> new ArchiveService(attachments, new AuditTrail()).archiveAndDelete("document", 1)) + .isInstanceOf(QException.class).hasMessageContaining("custom visibility policy"); + assertThat(ArchiveService.member(RecordFilesTables.FILE, file.id())).isNull(); + assertThat(attachments.list("document", 1)).hasSize(1); + + table.getCustomizers().remove(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.POST_QUERY_RECORD.getRole()); + for(var role : List.of(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_INSERT_RECORD, + com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.POST_INSERT_RECORD, + com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_UPDATE_RECORD, + com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.POST_UPDATE_RECORD, + com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.POST_DELETE_RECORD)) + { + table.withCustomizer(role, new com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference(NoOpCustomizer.class)); + } + assertThat(new ArchiveService(attachments, new AuditTrail()).archiveAndDelete("document", 1)).isPositive(); + assertThat(files.get(file.id()).recordId()).isEqualTo("1"); + } + + + + /******************************************************************************* + ** Host customizer with the interface's default (pass-through) behavior for every role. + *******************************************************************************/ + public static class NoOpCustomizer implements com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface + { + } + + + /******************************************************************************* ** Archived originals remain accessible through standard attachment reads and the exact reader ** after context restart. diff --git a/src/test/java/com/kingsrook/qbits/recordfiles/FixtureEndpoint.java b/src/test/java/com/kingsrook/qbits/recordfiles/FixtureEndpoint.java new file mode 100644 index 0000000..defa865 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/recordfiles/FixtureEndpoint.java @@ -0,0 +1,214 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.recordfiles; + + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.github.dockerjava.api.command.InspectContainerResponse; +import com.github.dockerjava.api.model.ExposedPort; +import com.github.dockerjava.api.model.PortBinding; +import com.github.dockerjava.api.model.Ports; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.GenericContainer; + + +/******************************************************************************* + ** Where a disposable test container is reachable FROM THIS JVM. + ** + ** The original behavior is Testcontainers' own: the host it reports (which honors + ** TESTCONTAINERS_HOST_OVERRIDE) and the published port. The bridge address of the + ** sibling is used in exactly one topology: a JVM inside a build container that the + ** SAME daemon reports on its default bridge (a local full verify through the host + ** socket). Being inside some container is not enough: on the native runner the job + ** container is not a container of the isolated test daemon at all, so a sibling's + ** bridge address (a private address of that other daemon) is unreachable and the + ** published endpoint at the configured host must be dialed. + ** + ** The choice is made from the daemon's own description of the build container's + ** networks (no connection probing). If the build container cannot be identified or + ** is not on the default bridge (a job network, host networking, a remote or + ** separate daemon), the published endpoint is used. Timeouts, scenarios and + ** assertions are unchanged. The address is re-read from the daemon on every call + ** (a restarted container may get another bridge IP or published port). + *******************************************************************************/ +final class FixtureEndpoint +{ + private static final String DEFAULT_BRIDGE = "bridge"; + + private static final Pattern MOUNTINFO_CONTAINER_ID = Pattern.compile("/docker/containers/([0-9a-f]{64})/"); + + private static volatile Boolean buildContainerOnDefaultBridge; + + + + /*************************************************************************** + ** Host and port to dial. + ***************************************************************************/ + record Address(String host, int port) + { + } + + + + /*************************************************************************** + ** Utility class. + ***************************************************************************/ + private FixtureEndpoint() + { + } + + + + /*************************************************************************** + ** Resolve the endpoint of one exposed port of a running container, from a fresh + ** inspection (not Testcontainers' cached mapping). + ***************************************************************************/ + static Address of(GenericContainer container, int internalPort) + { + InspectContainerResponse inspected = container.getDockerClient().inspectContainerCmd(container.getContainerId()).exec(); + var networks = inspected.getNetworkSettings().getNetworks(); + String bridgeIp = networks == null || networks.get(DEFAULT_BRIDGE) == null ? null : networks.get(DEFAULT_BRIDGE).getIpAddress(); + Integer publishedPort = null; + var bindings = inspected.getNetworkSettings().getPorts().getBindings().get(new ExposedPort(internalPort)); + if(bindings != null && bindings.length > 0 && bindings[0].getHostPortSpec() != null) + { + publishedPort = Integer.valueOf(bindings[0].getHostPortSpec()); + } + return (choose(onBridgeWithBuildContainer(), bridgeIp, internalPort, container.getHost(), publishedPort)); + } + + + + /*************************************************************************** + ** Pure decision, separately tested. With the build container on the default + ** bridge: the sibling's bridge address and its own port. Otherwise the published + ** endpoint exactly as Testcontainers reports it. + ***************************************************************************/ + static Address choose(boolean buildContainerOnDefaultBridge, String bridgeIp, int internalPort, String publishedHost, Integer publishedPort) + { + if(buildContainerOnDefaultBridge) + { + if(bridgeIp == null || bridgeIp.isBlank()) + { + throw new IllegalStateException("FIXTURE_ENDPOINT_NO_BRIDGE_ADDRESS: the build container cannot reach a sibling without its bridge address"); + } + return (new Address(bridgeIp, internalPort)); + } + if(publishedHost == null || publishedHost.isBlank() || publishedPort == null) + { + throw new IllegalStateException("FIXTURE_ENDPOINT_NOT_PUBLISHED: no published endpoint to dial"); + } + return (new Address(publishedHost, publishedPort)); + } + + + + /*************************************************************************** + ** How the fixture publishes its port: the historical loopback-only binding + ** wherever a loopback dial can work, otherwise the daemon's own default binding. + ***************************************************************************/ + static PortBinding publishedBinding(int internalPort) + { + boolean loopback = loopbackPublication(onBridgeWithBuildContainer(), DockerClientFactory.instance().dockerHostIpAddress()); + return (new PortBinding(loopback ? Ports.Binding.bindIpAndPort("127.0.0.1", 0) : Ports.Binding.empty(), new ExposedPort(internalPort))); + } + + + + /*************************************************************************** + ** Pure decision, separately tested. A loopback-only publication is reachable + ** when the build container dials the bridge address (publication unused) or the + ** Docker host is this machine; if Testcontainers dials another host (a remote or + ** separate daemon, TESTCONTAINERS_HOST_OVERRIDE) a loopback binding could never + ** be reached, so the daemon's configured default binding is used instead. + ***************************************************************************/ + static boolean loopbackPublication(boolean buildContainerOnDefaultBridge, String dockerHost) + { + return (buildContainerOnDefaultBridge || dockerHost == null || dockerHost.isBlank() + || dockerHost.equals("localhost") || dockerHost.equals("127.0.0.1") || dockerHost.equals("::1")); + } + + + + /*************************************************************************** + ** Whether this JVM runs in a container that the daemon reports on the default + ** bridge. Decided once per JVM from the daemon's description of this container. + ***************************************************************************/ + private static boolean onBridgeWithBuildContainer() + { + Boolean known = buildContainerOnDefaultBridge; + if(known == null) + { + known = decideBuildContainerOnDefaultBridge(); + buildContainerOnDefaultBridge = known; + } + return (known); + } + + + + /*************************************************************************** + ** Inspect the build container; anything that cannot be established means the + ** original published endpoint (false). + ***************************************************************************/ + private static boolean decideBuildContainerOnDefaultBridge() + { + if(!Files.exists(Path.of("/.dockerenv"))) + { + return (false); + } + try + { + String id = ownContainerId(Files.readString(Path.of("/proc/self/mountinfo")), System.getenv("HOSTNAME")); + if(id == null) + { + return (false); + } + var networks = DockerClientFactory.instance().client().inspectContainerCmd(id).exec().getNetworkSettings().getNetworks(); + return (networks != null && networks.containsKey(DEFAULT_BRIDGE)); + } + catch(Exception e) + { + return (false); + } + } + + + + /*************************************************************************** + ** The id of the container this JVM runs in: from the mount table when it names + ** the container directory, otherwise the host name (the container id unless + ** overridden); null when neither exists. + ***************************************************************************/ + static String ownContainerId(String mountInfo, String hostName) + { + Matcher matcher = MOUNTINFO_CONTAINER_ID.matcher(mountInfo == null ? "" : mountInfo); + if(matcher.find()) + { + return (matcher.group(1)); + } + return (hostName == null || hostName.isBlank() ? null : hostName.trim()); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/recordfiles/FixtureEndpointTest.java b/src/test/java/com/kingsrook/qbits/recordfiles/FixtureEndpointTest.java new file mode 100644 index 0000000..6b0e236 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/recordfiles/FixtureEndpointTest.java @@ -0,0 +1,114 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.recordfiles; + + +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/******************************************************************************* + ** The topology decision of the sibling-container fixtures. + *******************************************************************************/ +class FixtureEndpointTest +{ + private static final String CONTAINER_ID = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + + + + /*************************************************************************** + ** Build container on the default bridge: dial the sibling bridge IP and its own port, never the published port. + ***************************************************************************/ + @Test + void buildContainerOnTheBridgeUsesTheBridgeAddress() + { + assertEquals(new FixtureEndpoint.Address("172.17.0.9", 5432), FixtureEndpoint.choose(true, "172.17.0.9", 5432, "localhost", 41111)); + assertEquals(new FixtureEndpoint.Address("172.17.0.9", 5432), FixtureEndpoint.choose(true, "172.17.0.9", 5432, null, null)); + } + + + + /*************************************************************************** + ** Every other topology (JVM on the Docker host, a separate or remote daemon such as the native runner's isolated + ** test daemon, job network, host networking, unidentified build container) dials the published endpoint + ** at the host Testcontainers reports (including TESTCONTAINERS_HOST_OVERRIDE), never a private bridge address. + ***************************************************************************/ + @Test + void everyOtherTopologyDialsThePublishedEndpoint() + { + assertEquals(new FixtureEndpoint.Address("localhost", 41111), FixtureEndpoint.choose(false, "172.17.0.9", 5432, "localhost", 41111)); + assertEquals(new FixtureEndpoint.Address("172.17.0.1", 32768), FixtureEndpoint.choose(false, "172.17.0.3", 6379, "172.17.0.1", 32768)); + assertEquals(new FixtureEndpoint.Address("host.docker.internal", 40000), FixtureEndpoint.choose(false, null, 5432, "host.docker.internal", 40000)); + } + + + + /*************************************************************************** + ** An unknown endpoint is an error, never a guessed address. + ***************************************************************************/ + @Test + void missingEndpointFailsClosed() + { + assertThrows(IllegalStateException.class, () -> FixtureEndpoint.choose(true, null, 5432, "localhost", 41111)); + assertThrows(IllegalStateException.class, () -> FixtureEndpoint.choose(true, " ", 5432, "localhost", 41111)); + assertThrows(IllegalStateException.class, () -> FixtureEndpoint.choose(false, "172.17.0.9", 5432, "localhost", null)); + assertThrows(IllegalStateException.class, () -> FixtureEndpoint.choose(false, "172.17.0.9", 5432, " ", 41111)); + } + + + + /*************************************************************************** + ** The historical loopback-only publication stays wherever a loopback dial can work; when Testcontainers dials + ** another host (the native runner's separate daemon) the daemon's default binding is used so the dial can succeed. + ***************************************************************************/ + @Test + void loopbackPublicationOnlyWhereALoopbackDialCanWork() + { + assertTrue(FixtureEndpoint.loopbackPublication(false, "localhost")); + assertTrue(FixtureEndpoint.loopbackPublication(false, "127.0.0.1")); + assertTrue(FixtureEndpoint.loopbackPublication(false, "::1")); + assertTrue(FixtureEndpoint.loopbackPublication(false, null)); + assertTrue(FixtureEndpoint.loopbackPublication(true, "172.17.0.1")); + assertFalse(FixtureEndpoint.loopbackPublication(false, "172.17.0.1")); + assertFalse(FixtureEndpoint.loopbackPublication(false, "host.docker.internal")); + assertFalse(FixtureEndpoint.loopbackPublication(false, "docker.example.test")); + } + + + + /*************************************************************************** + ** The build container id comes from the mount table when present, else the host name. + ***************************************************************************/ + @Test + void ownContainerIdPrefersTheMountTableThenTheHostName() + { + String mountInfo = "700 699 0:50 /docker/containers/" + CONTAINER_ID + "/hostname /etc/hostname rw,relatime - ext4 /dev/sda1 rw"; + assertEquals(CONTAINER_ID, FixtureEndpoint.ownContainerId(mountInfo, "unrelated")); + assertEquals("abc123", FixtureEndpoint.ownContainerId("no container mounts here", " abc123 ")); + assertNull(FixtureEndpoint.ownContainerId("", null)); + assertNull(FixtureEndpoint.ownContainerId(null, " ")); + } + +} diff --git a/src/test/java/com/kingsrook/qbits/recordfiles/PostgresIntakeAndRetentionTest.java b/src/test/java/com/kingsrook/qbits/recordfiles/PostgresIntakeAndRetentionTest.java index 6f58af3..98e26ef 100644 --- a/src/test/java/com/kingsrook/qbits/recordfiles/PostgresIntakeAndRetentionTest.java +++ b/src/test/java/com/kingsrook/qbits/recordfiles/PostgresIntakeAndRetentionTest.java @@ -48,6 +48,16 @@ class PostgresIntakeAndRetentionTest extends IntakeAndRetentionTest + /******************************************************************************* + ** Observe the exact disposable provider after each intake case. + *******************************************************************************/ + @org.junit.jupiter.api.AfterEach + void reportFixtureHealth() + { + PostgresRecordFilesSupport.reportHealth(); + } + + /******************************************************************************* ** Remove the provider created by this class. *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qbits/recordfiles/PostgresRecordFilesSupport.java b/src/test/java/com/kingsrook/qbits/recordfiles/PostgresRecordFilesSupport.java index 61d94af..8bd7708 100644 --- a/src/test/java/com/kingsrook/qbits/recordfiles/PostgresRecordFilesSupport.java +++ b/src/test/java/com/kingsrook/qbits/recordfiles/PostgresRecordFilesSupport.java @@ -30,9 +30,6 @@ import java.util.ArrayList; import java.util.List; import java.util.TimeZone; import java.util.UUID; -import com.github.dockerjava.api.model.ExposedPort; -import com.github.dockerjava.api.model.PortBinding; -import com.github.dockerjava.api.model.Ports; import com.kingsrook.qbits.collaboration.attachments.uploadjournal.UploadJournalTable; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.module.postgres.model.metadata.PostgreSQLBackendMetaData; @@ -85,12 +82,11 @@ public final class PostgresRecordFilesSupport .withCommand("postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on") .withExposedPorts(5432) .withCreateContainerCmdModifier(command -> command.getHostConfig().withMemory(768L * 1024 * 1024) - .withPortBindings(new PortBinding(Ports.Binding.bindIpAndPort("127.0.0.1", 0), new ExposedPort(5432)))) + .withPortBindings(FixtureEndpoint.publishedBinding(5432))) .waitingFor(Wait.forLogMessage(".*database system is ready to accept connections.*\\s", 2).withStartupTimeout(Duration.ofMinutes(2))); postgres.start(); - var inspected = postgres.getDockerClient().inspectContainerCmd(postgres.getContainerId()).exec(); - var binding = inspected.getNetworkSettings().getPorts().getBindings().get(new ExposedPort(5432))[0]; - server = "jdbc:postgresql://127.0.0.1:" + binding.getHostPortSpec() + "/"; + var address = FixtureEndpoint.of(postgres, 5432); + server = "jdbc:postgresql://" + address.host() + ":" + address.port() + "/"; try(Connection connection = root(); var statement = connection.createStatement()) { statement.execute("CREATE ROLE " + APP_ROLE + " LOGIN PASSWORD '" + APP_PASSWORD + "' NOSUPERUSER NOCREATEDB NOCREATEROLE"); @@ -100,6 +96,29 @@ public final class PostgresRecordFilesSupport + /******************************************************************************* + ** Bounded fixture diagnosis without environment, SQL, credentials or payloads. + *******************************************************************************/ + public static void reportHealth() + { + try + { + var inspection = postgres.getDockerClient().inspectContainerCmd(postgres.getContainerId()).exec(); + var state = inspection.getState(); + var address = FixtureEndpoint.of(postgres, 5432); + boolean endpointMatches = server.equals("jdbc:postgresql://" + address.host() + ":" + address.port() + "/"); + org.apache.logging.log4j.LogManager.getLogger(PostgresRecordFilesSupport.class).warn( + "POSTGRES_FIXTURE_HEALTH running={} oom={} restarting={} endpointMatches={}", + state.getRunning(), state.getOOMKilled(), state.getRestarting(), endpointMatches); + } + catch(Exception failure) + { + org.apache.logging.log4j.LogManager.getLogger(PostgresRecordFilesSupport.class).warn( + "POSTGRES_FIXTURE_HEALTH inspectionFailure={}", failure.getClass().getSimpleName()); + } + } + + /******************************************************************************* ** Remove only this exact disposable container and restore the JVM zone. *******************************************************************************/ diff --git a/src/test/java/com/kingsrook/qbits/recordfiles/PostgresRecordFilesTest.java b/src/test/java/com/kingsrook/qbits/recordfiles/PostgresRecordFilesTest.java index 22d6242..56d7fec 100644 --- a/src/test/java/com/kingsrook/qbits/recordfiles/PostgresRecordFilesTest.java +++ b/src/test/java/com/kingsrook/qbits/recordfiles/PostgresRecordFilesTest.java @@ -29,9 +29,6 @@ import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.UUID; -import com.github.dockerjava.api.model.ExposedPort; -import com.github.dockerjava.api.model.PortBinding; -import com.github.dockerjava.api.model.Ports; import com.kingsrook.qbits.recordfiles.audit.AuditTrail; import com.kingsrook.qbits.recordfiles.audit.RecordFilesEvent; import com.kingsrook.qbits.recordfiles.audit.RecordFilesEventType; @@ -87,7 +84,7 @@ class PostgresRecordFilesTest extends RecordFilesFixture .withCommand("postgres", "-c", "fsync=on", "-c", "synchronous_commit=on", "-c", "full_page_writes=on") .withExposedPorts(5432) .withCreateContainerCmdModifier(command -> command.getHostConfig().withMemory(768L * 1024 * 1024) - .withPortBindings(new PortBinding(Ports.Binding.bindIpAndPort("127.0.0.1", 0), new ExposedPort(5432)))) + .withPortBindings(FixtureEndpoint.publishedBinding(5432))) .waitingFor(Wait.forLogMessage(".*database system is ready to accept connections.*\\s", 2).withStartupTimeout(Duration.ofMinutes(2))); postgres.start(); refreshEndpoint(); @@ -647,8 +644,7 @@ class PostgresRecordFilesTest extends RecordFilesFixture ***************************************************************************/ private static void refreshEndpoint() { - var inspected = postgres.getDockerClient().inspectContainerCmd(postgres.getContainerId()).exec(); - var binding = inspected.getNetworkSettings().getPorts().getBindings().get(new ExposedPort(5432))[0]; - server = "jdbc:postgresql://127.0.0.1:" + binding.getHostPortSpec() + "/"; + var address = FixtureEndpoint.of(postgres, 5432); + server = "jdbc:postgresql://" + address.host() + ":" + address.port() + "/"; } }