ci: use the shared branch-aware CodeQL worker

Co-Authored-By: Codex <noreply@openai.com>

.circleci/config.yml

@@ -1,65 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-  qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-  configure-package-install:
-    steps:
-      - run:
-          name: Keep package installation unattended
-          command: |
-            # These build machines are discarded after the job.
-            sudo install -d /etc/needrestart/conf.d
-            printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.12
 workflows:
-
-  test_only:
+  setup:
     jobs:
-      - qqq-orb/mvn_test_only:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          filters:
-            branches:
-              ignore: /(develop|release/.*|integration.*)/
-            tags:
-              only: []
-
-  publish_snapshot:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: snapshot
-          filters:
-            branches:
-              only:
-                - develop
-
-  publish_release_candidate:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: release_candidate
-          filters:
-            branches:
-              only:
-                - /release\/.*/
-
-  publish_release:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: release
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: [ /^v\d+\.\d+\.\d+$/ ]
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.12
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@17c94592d94c7d56c440277be6fe3526692887a6

.github/workflows/release-rc.yml

@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
         type: boolean
         default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
   pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
   contents: write
-
+  actions: read
+  checks: read
 jobs:
   release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
     with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: 0.7.12
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release-no-integration
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
 
 ## [Unreleased]
 
+### Fixed
+
+- Endpoint pickers for saved views, saved reports and saved bulk-load profiles are dropped when
+  the host has not registered those possible-value sources, so hosts without them validate.
+
 ### Added
 
 - File transfer Preferences with connect-permission visibility, read-only connection details,

README.md

@@ -36,6 +36,10 @@ Apply [docs/ddl/file_gateway.sql](docs/ddl/file_gateway.sql) to the host's datab
 H2 and PostgreSQL and stores UTC timestamps without a zone, matching QQQ's RDBMS binding.
 The gateway does not create or migrate tables at runtime.
 
+Register core's saved views, saved reports and saved bulk-load profiles (if the app uses them)
+before the gateway producer: endpoint pickers for a source the app hasn't registered become plain
+id fields, so the instance still validates.
+
 Register metadata while building the QInstance:
 
 ```java

docs/DESIGN.md

@@ -458,9 +458,11 @@ sized for the largest report. The no-disk test asserts the temp area is empty af
   (§15) would add only a server and an authenticator.
 
 ## 7. Audit
-- **Connections.** Every connection writes `file_gateway_connection_log`. The row is opened at TCP
-  accept and updated at auth and at disconnect. It records the outcome, IP, key fingerprint, SSH
-  client version, byte and operation counts, and channel `sftp`.
+- **Connections.** Every sign-in attempt writes `file_gateway_connection_log`. The row is opened
+  at the first authentication attempt (or a pre-auth refusal such as the connection rate limit)
+  and updated at auth and at disconnect. It records the outcome, IP, key fingerprint, SSH client
+  version, byte and operation counts, and channel `sftp`. A TCP connection that never tries to
+  sign in, such as a kubelet health probe or a port scan, leaves no row.
 - **Operations.** Every list, upload, download, rename and remove writes
   `file_gateway_transfer_log`. It records the operation, path, object kind and id, bytes, sha256,
   duration, outcome and message, with channel `sftp/<sessionId>`.

src/main/java/com/kingsrook/qbits/filegateway/FileGatewayQBitProducer.java

@@ -29,6 +29,7 @@ import com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData;
 import com.kingsrook.qqq.backend.core.exceptions.QException;
 import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
+import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection;
 import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
@@ -43,6 +44,9 @@ import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
  *******************************************************************************/
 public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGatewayQBitConfig>
 {
+   /** Core possible-value sources a host may or may not register. */
+   static final java.util.Set<String> OPTIONAL_HOST_SOURCES = java.util.Set.of("savedView", "savedReport", "savedBulkLoadProfile");
+
    /*******************************************************************************
     ** The sftp wall owns its list (core's qbit prefixes are transitional and the
     ** assistant may now read audits): partners never reach identity, audits,
@@ -149,6 +153,7 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
             table = qBitConfig.getTableMetaDataCustomizer().customizeMetaData(qInstance, table);
          }
          table.setSourceQBitName(qBit.getName());
+         dropMissingPickers(qInstance, table);
          output.add(table);
          if(FileGatewayTables.MAIN.contains(table.getName()))
          {
@@ -187,4 +192,25 @@ public class FileGatewayQBitProducer implements QBitMetaDataProducer<FileGateway
          new com.kingsrook.qbits.filegateway.storage.ObjectStorageGatewayStore(qInstance, qBitConfig.getFileGatewayBucketAlias()), qBitConfig, java.time.Clock.systemUTC()));
       return (new MetaDataProducerMultiOutput());
    }
+
+
+
+   /*******************************************************************************
+    ** Saved views, saved reports and saved bulk-load profiles are optional in a
+    ** host. A picker whose possible-value source the host has not registered (yet)
+    ** stays a plain id field, so the instance validates; register those core
+    ** metadata providers before this producer to get the pickers.
+    *******************************************************************************/
+   static void dropMissingPickers(QInstance qInstance, QTableMetaData table)
+   {
+      for(QFieldMetaData field : table.getFields().values())
+      {
+         String source = field.getPossibleValueSourceName();
+         if(source != null && OPTIONAL_HOST_SOURCES.contains(source) && qInstance.getPossibleValueSource(source) == null)
+         {
+            field.setPossibleValueSourceName(null);
+            field.setPossibleValueSourceFilter(null);
+         }
+      }
+   }
 }

src/main/java/com/kingsrook/qbits/filegateway/metadata/FileGatewayHome.java

@@ -108,13 +108,13 @@ public class FileGatewayHome extends AbstractWidgetRenderer
          long value = "transfers".equals(key) ? rows.size() : rows.stream()
             .filter(row -> ("in".equals(key) ? "UPLOAD" : "DOWNLOAD").equals(row.getValueString("operation")))
             .mapToLong(row -> number(row, "bytes")).sum();
-         return (new StatisticsData("transfers".equals(key) ? value : Math.round(value / 10485.76) / 100.0, null, null));
+         return (new StatisticsData("transfers".equals(key) ? value : Math.round(value / 10485.76) / 100.0, null, null).withIncreaseIsGood(true));
       }
       if("locked".equals(key))
       {
          long count = query(FileGatewayTables.ACCOUNT, new QQueryFilter()).stream().filter(row -> "LOCKED".equals(row.getValueString("status"))
             || (row.getValueInstant("lockedUntil") != null && row.getValueInstant("lockedUntil").isAfter(now))).count();
-         return (new StatisticsData(count, null, null));
+         return (new StatisticsData(count, null, null).withIncreaseIsGood(false));
       }
       if(List.of("failures", "recent").contains(key))
       {
@@ -122,24 +122,26 @@ public class FileGatewayHome extends AbstractWidgetRenderer
             .withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, "FAILED", "PARTIAL", "REJECTED")));
          if("failures".equals(key))
          {
-            return (new StatisticsData(rows.size(), null, null));
+            return (new StatisticsData(rows.size(), null, null).withIncreaseIsGood(false));
          }
          rows.sort(java.util.Comparator.comparing((QRecord row) -> row.getValueInstant("finishedAt")).reversed());
          return (table("Recent failures", FileGatewayTables.UPLOAD_JOB, rows.stream().limit(20).toList(),
-            List.of("fileName", "status", "errorMessage", "finishedAt"), List.of("File", "Status", "Message", "Finished")));
+            List.of("fileName", "status", "errorMessage", "finishedAt"), List.of("File", "Status", "Message", "Finished"), "No failed transfers in the last 7 days."));
       }
       if(List.of("sessions", "active").contains(key))
       {
          var rows = query(FileGatewayTables.CONNECTION_LOG, since("lastSeenAt", now.minusSeconds(180))
             .withCriteria(new QFilterCriteria("disconnectedAt", QCriteriaOperator.IS_BLANK))
             .withCriteria(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "CONNECTED")));
-         return ("sessions".equals(key) ? new StatisticsData(rows.size(), null, null) : table("Active sessions", FileGatewayTables.CONNECTION_LOG, rows,
-            List.of("username", "remoteIp", "connectedAt", "operationCount", "podName"), List.of("Username", "IP", "Connected", "Operations", "Pod")));
+         return ("sessions".equals(key) ? new StatisticsData(rows.size(), null, null).withIncreaseIsGood(true) : table("Active sessions", FileGatewayTables.CONNECTION_LOG, rows,
+            List.of("username", "remoteIp", "connectedAt", "operationCount"), List.of("Username", "IP", "Connected", "Operations"),
+            "No one is connected right now (no session has been seen in the last 3 minutes)."));
       }
       if("servers".equals(key))
       {
          return (table("Server", FileGatewayTables.SERVER, query(FileGatewayTables.SERVER, since("lastSeenAt", now.minusSeconds(180))),
-            List.of("podName", "hostKeyFingerprint", "port", "lastSeenAt"), List.of("Pod", "Host key fingerprint", "Port", "Last seen")));
+            List.of("podName", "hostKeyFingerprint", "port", "lastSeenAt"), List.of("Pod", "Host key fingerprint", "Port", "Last seen"),
+            "No gateway server has reported in during the last 3 minutes."));
       }
       Map<Integer, QRecord> partners = new LinkedHashMap<>();
       for(QRecord account : query(FileGatewayTables.ACCOUNT, new QQueryFilter()))
@@ -157,7 +159,8 @@ public class FileGatewayHome extends AbstractWidgetRenderer
       }
       List<QRecord> rows = partners.values().stream().filter(row -> number(row, "transfers") > 0)
          .sorted(java.util.Comparator.comparingLong((QRecord row) -> number(row, "transfers")).reversed()).limit(10).toList();
-      return (table("Top partners (7 days)", FileGatewayTables.ACCOUNT, rows, List.of("username", "transfers", "mb"), List.of("Partner", "Transfers", "MB")));
+      return (table("Top partners (7 days)", FileGatewayTables.ACCOUNT, rows, List.of("username", "transfers", "mb"), List.of("Partner", "Transfers", "MB"),
+         "No partner has transferred a file in the last 7 days."));
    }
 
 
@@ -165,12 +168,12 @@ public class FileGatewayHome extends AbstractWidgetRenderer
    /*******************************************************************************
     ** Typed text blocks carry record links; cells never contain markup.
     *******************************************************************************/
-   private TableData table(String label, String table, List<QRecord> records, List<String> fields, List<String> labels)
+   private TableData table(String label, String table, List<QRecord> records, List<String> fields, List<String> labels, String noRowsText)
    {
       List<TableData.Column> columns = new ArrayList<>();
       for(int i = 0; i < fields.size(); i++)
       {
-         columns.add(new TableData.Column(i == 0 ? "block" : "default", labels.get(i), fields.get(i), "1fr", null));
+         columns.add(new TableData.Column(i == 0 ? "block" : "default", labels.get(i), fields.get(i), null, null));
       }
       List<Map<String, Object>> rows = new ArrayList<>();
       for(QRecord record : records)
@@ -185,7 +188,7 @@ public class FileGatewayHome extends AbstractWidgetRenderer
             .withLink("/record/" + table + "/" + record.getValue("id")));
          rows.add(row);
       }
-      return (new TableData(null, columns, rows).withRowsPerPage(10).withHidePaginationDropdown(true));
+      return (new TableData(null, columns, rows).withRowsPerPage(10).withHidePaginationDropdown(true).withNoRowsFoundHTML(noRowsText));
    }
 
 

src/main/java/com/kingsrook/qbits/filegateway/permissions/FileGatewayPermissionChecker.java

@@ -22,14 +22,19 @@ package com.kingsrook.qbits.filegateway.permissions;
 
 
 import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
+import com.kingsrook.qqq.backend.core.actions.permissions.AvailablePermission;
 import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionSubType;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
 import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
 import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
+import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
+import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
 import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
 import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
@@ -60,4 +65,42 @@ public class FileGatewayPermissionChecker implements CustomPermissionChecker
          throw (new QPermissionDeniedException("Missing file gateway permission"));
       }
    }
+
+
+
+   /*******************************************************************************
+    ** This checker gates on the three gateway permission names, not on per-object
+    ** names, so it tells the catalog (getAllAvailablePermissions) those names.
+    *******************************************************************************/
+   @Override
+   public boolean handlesBuildAvailablePermission()
+   {
+      return (true);
+   }
+
+
+
+   /*******************************************************************************
+    ** Collapse every gateway object onto file_gateway.view, .admin or .connect.
+    ** The object name and type are fixed so the catalog (a set) lists each name
+    ** once: reads and the app need view, writes and processes need admin, and the
+    ** app also stands for SFTP sign-in (connect), which the gateway grants to
+    ** sessions rather than checking on an object.
+    *******************************************************************************/
+   @Override
+   public AvailablePermission buildAvailablePermission(QPermissionRules rules, PermissionSubType permissionSubType, String baseName, MetaDataWithName metaDataWithName, String objectType)
+   {
+      AvailablePermission base = buildBaseAvailablePermission(rules, permissionSubType, metaDataWithName, objectType);
+      if(base == null)
+      {
+         return (null);
+      }
+      boolean write = "WRITE".equals(base.getPermissionType()) || "Process".equals(objectType);
+      String  name  = write ? FileGatewayPermissions.ADMIN : FileGatewayPermissions.VIEW;
+      if(metaDataWithName instanceof QAppMetaData)
+      {
+         name = FileGatewayPermissions.CONNECT;
+      }
+      return (base.withName(name).withObjectName("File Gateway").withObjectType("App").withPermissionType(name.substring(name.indexOf('.') + 1)));
+   }
 }

src/main/java/com/kingsrook/qbits/filegateway/sftp/GatewaySession.java

@@ -48,6 +48,7 @@ public class GatewaySession
    public final ServerSession ssh;
    public final String remoteIp;
    public Integer connectionId;
+   public java.time.Instant connectedAt;
    public QRecord account;
    public QRecord key;
    public long operations;

src/main/java/com/kingsrook/qbits/filegateway/sftp/SshKeyAuthenticator.java

@@ -82,16 +82,15 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
 
 
    /*******************************************************************************
-    ** Audit TCP accept and enforce the pre-authentication connection rate.
+    ** Track TCP accept and enforce the pre-authentication connection rate. The
+    ** connection-log row is written at the first sign-in attempt (or a refusal),
+    ** so TCP health probes and port scans that never authenticate leave no row.
     *******************************************************************************/
    public synchronized void accepted(ServerSession ssh) throws QException
    {
       GatewaySession state = new GatewaySession(instance, ssh, clock);
       ssh.setAttribute(GatewaySession.ATTRIBUTE, state);
-      state.connectionId = audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord()
-         .withValue("sessionId", state.id).withValue("remoteIp", state.remoteIp).withValue("podName", podName)
-         .withValue("connectedAt", clock.instant()).withValue("lastSeenAt", clock.instant())
-         .withValue("clientChannel", "sftp").withValue("outcome", "AUTH_FAILED")).getValueInteger("id");
+      state.connectedAt = clock.instant();
       sessions.put(state.id, state);
       prune(connections, clock.instant().minusSeconds(60));
       Deque<Instant> window = connections.computeIfAbsent(state.remoteIp, ignored -> new ArrayDeque<>());
@@ -144,6 +143,7 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
       }
       QRecord account = first(FileGatewayTables.ACCOUNT, "username", username);
       String fingerprint = KeyUtils.getFingerPrint(BuiltinDigests.sha256, publicKey);
+      ensureConnectionRow(state);
       audit.update(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("id", state.connectionId)
          .withValue("username", username).withValue("keyFingerprint", fingerprint).withValue("clientVersion", ssh.getClientVersion())
          .withValue("accountId", account == null ? null : account.getValue("id")));
@@ -292,11 +292,30 @@ public class SshKeyAuthenticator implements PublickeyAuthenticator, FileGatewayA
 
 
 
+   /*******************************************************************************
+    ** Write the connection-log row on first need (an auth attempt or a refusal).
+    ** Its outcome stays AUTH_FAILED until the sign-in succeeds.
+    *******************************************************************************/
+   private void ensureConnectionRow(GatewaySession state) throws QException
+   {
+      if(state.connectionId == null)
+      {
+         Instant at = state.connectedAt == null ? clock.instant() : state.connectedAt;
+         state.connectionId = audit.insert(FileGatewayTables.CONNECTION_LOG, new QRecord()
+            .withValue("sessionId", state.id).withValue("remoteIp", state.remoteIp).withValue("podName", podName)
+            .withValue("connectedAt", at).withValue("lastSeenAt", clock.instant())
+            .withValue("clientChannel", "sftp").withValue("outcome", "AUTH_FAILED")).getValueInteger("id");
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Record the authentication outcome without credentials.
     *******************************************************************************/
    private boolean outcome(GatewaySession state, String outcome, String reason) throws QException
    {
+      ensureConnectionRow(state);
       audit.update(FileGatewayTables.CONNECTION_LOG, new QRecord().withValue("id", state.connectionId)
          .withValue("outcome", outcome).withValue("disconnectReason", reason).withValue("lastSeenAt", clock.instant()));
       if(reason != null)

src/test/java/com/kingsrook/qbits/filegateway/FileGatewayQBitProducerTest.java

@@ -22,6 +22,8 @@ package com.kingsrook.qbits.filegateway;
 
 import java.util.List;
 import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
+import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
+import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData;
 import org.junit.jupiter.api.Test;
@@ -73,6 +75,21 @@ class FileGatewayQBitProducerTest
 
 
 
+   /*******************************************************************************
+    ** The three gateway permission names reach QQQ's catalog, so identity
+    ** policies can grant them.
+    *******************************************************************************/
+   @Test
+   void permissionsAreInTheAvailableCatalog() throws Exception
+   {
+      var instance = new QInstance();
+      var config = new FileGatewayQBitConfig().withBackendName("host-db");
+      new FileGatewayQBitProducer().withQBitConfig(config).withNamespace("test").produce(instance).addSelfToInstance(instance);
+      assertThat(PermissionsHelper.getAllAvailablePermissionNames(instance)).containsAll(FileGatewayPermissions.ALL);
+   }
+
+
+
    /*******************************************************************************
     ** Package scanning never enables an unconfigured producer.
     *******************************************************************************/
@@ -100,6 +117,8 @@ class FileGatewayQBitProducerTest
    void contingentPickerMetadata(String source) throws Exception
    {
       var instance = new QInstance();
+      instance.addPossibleValueSource(new com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource().withName(source)
+         .withType(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType.TABLE).withTableName(source));
       new FileGatewayQBitProducer().withQBitConfig(new FileGatewayQBitConfig()).produce(instance);
       var field = instance.getTable(FileGatewayTables.ALIAS).getField(source + "Id");
       assertThat(field.getPossibleValueSourceName()).isEqualTo(source);
@@ -112,4 +131,27 @@ class FileGatewayQBitProducerTest
       });
       assertThat(filter.getSubFilters()).isNullOrEmpty();
    }
+
+
+
+   /*******************************************************************************
+    ** A host without saved views, reports or bulk-load profiles still validates:
+    ** those pickers become plain id fields; present sources keep their pickers.
+    *******************************************************************************/
+   @Test
+   void dropsPickersForMissingHostSources() throws Exception
+   {
+      QInstance instance = new QInstance();
+      com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData alias = FileGatewayTables.tables(new FileGatewayQBitConfig()).stream()
+         .filter(t -> t.getName().equals(FileGatewayTables.ALIAS)).findFirst().orElseThrow();
+      instance.addPossibleValueSource(new com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource().withName("savedView")
+         .withType(com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSourceType.TABLE).withTableName("savedView"));
+      FileGatewayQBitProducer.dropMissingPickers(instance, alias);
+      assertThat(alias.getField("savedViewId").getPossibleValueSourceName()).isEqualTo("savedView");
+      assertThat(alias.getField("savedViewId").getPossibleValueSourceFilter()).isNotNull();
+      assertThat(alias.getField("savedReportId").getPossibleValueSourceName()).isNull();
+      assertThat(alias.getField("savedReportId").getPossibleValueSourceFilter()).isNull();
+      assertThat(alias.getField("savedBulkLoadProfileId").getPossibleValueSourceName()).isNull();
+      assertThat(alias.getField("tableName").getPossibleValueSourceName()).isEqualTo("tables");
+   }
 }

src/test/java/com/kingsrook/qbits/filegateway/PhaseEIntegrationTest.java

@@ -274,6 +274,36 @@ class PhaseEIntegrationTest
          assertThat(((Number) ((StatisticsData) home.data("out", now)).getCount()).doubleValue()).isEqualTo(1);
          assertThat(((Number) ((StatisticsData) home.data("failures", now)).getCount()).intValue()).isEqualTo(1);
          assertThat(((TableData) home.data("recent", now)).getRows()).hasSize(1);
+         ///////////////////////////////////////////////////////////////////////////////
+         // direction: failures and locks are bad when they grow; volume is not bad  //
+         ///////////////////////////////////////////////////////////////////////////////
+         assertThat(((StatisticsData) home.data("failures", now)).getIncreaseIsGood()).as("failures").isFalse();
+         assertThat(((StatisticsData) home.data("locked", now)).getIncreaseIsGood()).as("locked").isFalse();
+         for(String key : List.of("transfers", "in", "out", "sessions"))
+         {
+            assertThat(((StatisticsData) home.data(key, now)).getIncreaseIsGood()).as(key).isTrue();
+         }
+         ///////////////////////////////////////////////////////////////////////////////
+         // every home table says why it is empty                                     //
+         ///////////////////////////////////////////////////////////////////////////////
+         for(String key : List.of("recent", "partners", "active", "servers"))
+         {
+            assertThat(((TableData) home.data(key, now)).getNoRowsFoundHTML()).as(key).isNotBlank();
+         }
+         assertThat(((TableData) home.data("active", now)).getRows()).isEmpty();
+         assertThat(((TableData) home.data("active", now)).getNoRowsFoundHTML()).contains("No one is connected");
+         //////////////////////////////////////////////////////////////////
+         // readable at all widths: home tables declare no column widths //
+         //////////////////////////////////////////////////////////////////
+         assertThat(((TableData) home.data("recent", now)).getColumns()).allSatisfy(column -> assertThat(column.getWidth()).isNull());
+         for(String key : List.of("recent", "partners", "active", "servers"))
+         {
+            ////////////////////////////////////////////////////////////////////////
+            // at most four columns, so tables stay readable at phone widths     //
+            ////////////////////////////////////////////////////////////////////////
+            assertThat(((TableData) home.data(key, now)).getColumns()).as(key).hasSizeLessThanOrEqualTo(4)
+               .allSatisfy(column -> assertThat(column.getWidth()).isNull());
+         }
          assertThat(((TableData) home.data("partners", now)).getRows()).hasSize(1);
          for(String widget : FileGatewayHome.KEYS)
          {

src/test/java/com/kingsrook/qbits/filegateway/SftpIntegrationTest.java

@@ -78,6 +78,35 @@ class SftpIntegrationTest
 
 
 
+   /*******************************************************************************
+    ** A TCP connection that never signs in (a health probe or port scan) leaves
+    ** no connection-log row and no failure; a real sign-in attempt does.
+    *******************************************************************************/
+   @Test
+   void tcpProbeLeavesNoConnectionRow() throws Exception
+   {
+      try(var h = new FileGatewayTestServer())
+      {
+         for(int i = 0; i < 3; i++)
+         {
+            try(java.net.Socket socket = new java.net.Socket("127.0.0.1", h.runtime.getProtocol().getPort()))
+            {
+               socket.setSoTimeout(2000);
+               socket.getInputStream().read();
+            }
+         }
+         Thread.sleep(300);
+         assertThat(h.audit.query("file_gateway_connection_log", new QQueryFilter())).isEmpty();
+
+         KeyPair unknown = pair();
+         org.assertj.core.api.Assertions.assertThatThrownBy(() -> h.login("nobody", unknown)).isInstanceOf(Exception.class);
+         assertThat(h.audit.query("file_gateway_connection_log", new QQueryFilter())).singleElement()
+            .satisfies(row -> assertThat(row.getValueString("outcome")).isEqualTo("AUTH_FAILED"));
+      }
+   }
+
+
+
    /*******************************************************************************
     ** Permission filtered listings and audit.
     *******************************************************************************/