ci: centralize QBit source scanning

Co-Authored-By: Codex <noreply@openai.com>

.circleci/config.yml

@@ -1,65 +1,14 @@
 version: 2.1
-
+setup: true
 orbs:
-  qqq-orb: kingsrook/qqq-orb@0.6.9
-
-commands:
-  configure-package-install:
-    steps:
-      - run:
-          name: Keep package installation unattended
-          command: |
-            # These build machines are discarded after the job.
-            sudo install -d /etc/needrestart/conf.d
-            printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
-
+  qqq-orb: kingsrook/qqq-orb@0.7.12
 workflows:
-
-  test_only:
+  setup:
     jobs:
-      - qqq-orb/mvn_test_only:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          filters:
-            branches:
-              ignore: /(develop|release/.*|integration.*)/
-            tags:
-              only: []  
-
-  publish_snapshot:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: snapshot
-          filters:
-            branches:
-              only:
-                - develop
-
-  publish_release_candidate:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: release_candidate
-          filters:
-            branches:
-              only:
-                - /release\/.*/
-
-  publish_release:
-    jobs:
-      - qqq-orb/mvn_publish:
-          pre-steps:
-            - configure-package-install
-          context: [ qqq-maven-registry-credentials ]
-          branch_type: release
-          filters:
-            branches:
-              ignore: /.*/
-            tags:
-              only: [ /^v\d+\.\d+\.\d+$/ ]
+    - qqq-orb/generate_pipeline:
+        orb_version: 0.7.12
+        filters:
+          branches:
+            ignore: automation/qbit-rc-pending
+          tags:
+            only: /^v\d+\.\d+\.\d+$/

.github/workflows/codeql.yml

@@ -0,0 +1,23 @@
+name: CodeQL
+'on':
+  push:
+    branches:
+    - main
+    - develop
+    - release/**
+    - hotfix/**
+  pull_request: {}
+  schedule:
+  - cron: 41 16 * * 0
+  workflow_dispatch: {}
+permissions:
+  contents: read
+  actions: read
+  security-events: write
+  packages: read
+concurrency:
+  group: codeql-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+jobs:
+  analyze:
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22

.github/workflows/release-rc.yml

@@ -1,25 +1,74 @@
-# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
-# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
-# the release line and the release/* push runs CircleCI publish_release_candidate.
 name: Release candidate
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       dry_run:
-        description: Print the plan without merging or creating branches
+        description: Plan a cut without changing release state
         type: boolean
         default: false
+      finalize:
+        description: Retry finalization of an existing cut; do not cut another RC
+        type: boolean
+        default: false
+      recover:
+        description: Recover the same published RC after a deployment timeout; never
+          reupload
+        type: boolean
+        default: false
+      circle_build:
+        description: Original Circle publisher build number (recovery only)
+        type: string
+        default: ''
+      merge_sha:
+        description: Existing cut merge_sha (finalize retry only)
+        type: string
+        default: ''
+      bump_sha:
+        description: Existing cut bump_sha (finalize retry only)
+        type: string
+        default: ''
+      branch:
+        description: Existing cut branch (finalize retry only)
+        type: string
+        default: ''
+      version:
+        description: Existing cut version (finalize retry only)
+        type: string
+        default: ''
   pull_request:
-    types: [closed]
-    branches: [develop]
-
+    types:
+    - closed
+    branches:
+    - develop
+  repository_dispatch:
+    types:
+    - qbit-rc-published
 permissions:
   contents: write
-
+  actions: read
+  checks: read
 jobs:
   release-rc:
-    if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
-    uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
+    if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
+      || (github.event_name == 'pull_request' && github.event.pull_request.merged
+      == true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
+      'release:rc'))
+    uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
     with:
-      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
+      dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
+        }}
+      finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
+        == 'workflow_dispatch' && inputs.finalize) }}
+      recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
+        }}
+      circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
+        || '' }}
+      merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
+        || inputs.merge_sha || '' }}
+      bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
+        || inputs.bump_sha || '' }}
+      branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
+        || inputs.branch || '' }}
+      version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
+        || inputs.version || '' }}
+    secrets: inherit

.qqq-ci.yml

@@ -0,0 +1,16 @@
+profile: qbit
+orb_version: 0.7.12
+default_branch: develop
+java_version: '21'
+browsers: true
+api_checks: false
+docs_only: false
+release_on_main: false
+release_tag_policy: stable
+test_branch_policy: non-release-no-integration
+publish:
+  snapshot: true
+  release_candidate: true
+  release: true
+  asciidoc: false
+release_completion: true

src/main/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeView.java

@@ -22,11 +22,18 @@ package com.kingsrook.qbits.webhooks.app;
 
 
 import java.io.Serializable;
+import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.util.Locale;
+import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.AttemptOutcome;
+import com.kingsrook.qbits.webhooks.metadata.WebhooksEnums.DeliveryStatus;
 import com.kingsrook.qbits.webhooks.metadata.WebhooksTables;
+import com.kingsrook.qbits.webhooks.support.FilterLinks;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.utils.ValueUtils;
 
@@ -117,10 +124,18 @@ public final class WebhooksHomeView
    private static void kpis(StringBuilder html, WebhooksHomeData data, String prefix)
    {
       html.append("<div style=\"").append(KPIS).append("\">");
-      kpi(html, String.valueOf(data.getDelivered24h()), "Delivered (24 h)", false, prefix + "/" + WebhooksTables.DELIVERY);
-      kpi(html, String.valueOf(data.getFailedAttempts24h()), "Failed attempts (24 h)", data.getFailedAttempts24h() > 0, prefix + "/" + WebhooksTables.DELIVERY);
-      kpi(html, String.valueOf(data.getDeadLetters()), "Dead letters", data.getDeadLetters() > 0, prefix + "/" + WebhooksTables.DELIVERY);
-      kpi(html, String.valueOf(data.getBacklog()), "Waiting to send", false, prefix + "/" + WebhooksTables.DELIVERY);
+      String   delivery = prefix + "/" + WebhooksTables.DELIVERY;
+      String   since    = data.getNow().minus(Duration.ofHours(24)).toString();
+      kpi(html, String.valueOf(data.getDelivered24h()), "Delivered (24 h)", false,
+         FilterLinks.href(delivery, new QQueryFilter(new QFilterCriteria("deliveredDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, since))));
+      kpi(html, String.valueOf(data.getFailedAttempts24h()), "Failed attempts (24 h)", data.getFailedAttempts24h() > 0,
+         FilterLinks.href(prefix + "/" + WebhooksTables.ATTEMPT, new QQueryFilter(new QFilterCriteria("startDate", QCriteriaOperator.GREATER_THAN_OR_EQUALS, since),
+            new QFilterCriteria("outcome", QCriteriaOperator.NOT_IN, AttemptOutcome.SUCCEEDED.name(), AttemptOutcome.RATE_LIMITED.name(), AttemptOutcome.IN_FLIGHT.name()))));
+      kpi(html, String.valueOf(data.getDeadLetters()), "Dead letters", data.getDeadLetters() > 0,
+         FilterLinks.href(delivery, new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.EQUALS, DeliveryStatus.DEAD_LETTER.name()))));
+      kpi(html, String.valueOf(data.getBacklog()), "Waiting to send", false,
+         FilterLinks.href(delivery, new QQueryFilter(new QFilterCriteria("status", QCriteriaOperator.IN,
+            DeliveryStatus.PENDING.name(), DeliveryStatus.RETRY_WAIT.name(), DeliveryStatus.SENDING.name()))));
       kpi(html, data.getP50Millis() == null ? "None yet" : data.getP50Millis() + " / " + data.getP95Millis() + " ms", "Latency, median / 95th (24 h)", false, null);
       html.append("</div>");
    }

src/main/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidget.java

@@ -23,7 +23,7 @@ package com.kingsrook.qbits.webhooks.health;
 
 import java.util.List;
 import java.util.Set;
-import com.kingsrook.qqq.backend.core.actions.ai.output.AiLinkBuilder;
+import com.kingsrook.qbits.webhooks.support.FilterLinks;
 import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
 import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
 import com.kingsrook.qqq.backend.core.context.QContext;
@@ -64,7 +64,7 @@ public class WebhooksHealthWidget extends AbstractWidgetRenderer
          }
       }
       return (new RenderWidgetOutput(new StatisticsData().withCount(status).withCountContext("Webhooks - System Health")
-         .withCountURL(AiLinkBuilder.filteredTableHref(HealthMetaDataProvider.CHECK_TABLE,
+         .withCountURL(FilterLinks.href("/app/" + HealthMetaDataProvider.CHECK_TABLE,
             new QQueryFilter(new QFilterCriteria("component", QCriteriaOperator.EQUALS, "webhooks"))))));
    }
 }

src/main/java/com/kingsrook/qbits/webhooks/support/FilterLinks.java

@@ -0,0 +1,59 @@
+/*
+ * QQQ - Low-code Application Framework for Engineers.
+ * Copyright (C) 2021-2025.  Kingsrook, LLC
+ * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
+ * contact@kingsrook.com
+ * https://github.com/Kingsrook/
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.kingsrook.qbits.webhooks.support;
+
+
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
+
+
+/*******************************************************************************
+ ** Links to a filtered table, in the form every QQQ qbit uses and the Next UI
+ ** reads: {@code <path>?filter=<url-encoded JSON QQueryFilter>}.
+ *******************************************************************************/
+public final class FilterLinks
+{
+   /*******************************************************************************
+    ** Utility class.
+    *******************************************************************************/
+   private FilterLinks()
+   {
+   }
+
+
+
+   /*******************************************************************************
+    ** The path (a table path, with any prefix), with the filter as URL-encoded JSON.
+    *******************************************************************************/
+   public static String href(String path, QQueryFilter filter)
+   {
+      Map<String, Object> link = new LinkedHashMap<>();
+      link.put("criteria", filter.getCriteria());
+      link.put("orderBys", filter.getOrderBys());
+      link.put("subFilters", filter.getSubFilters());
+      link.put("booleanOperator", filter.getBooleanOperator());
+      return (path + "?filter=" + URLEncoder.encode(JsonUtils.toJson(link), StandardCharsets.UTF_8));
+   }
+}

src/test/java/com/kingsrook/qbits/webhooks/app/WebhooksHomeTest.java

@@ -21,7 +21,12 @@
 package com.kingsrook.qbits.webhooks.app;
 
 
+import java.net.URLDecoder;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
 import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import com.kingsrook.qbits.webhooks.WebhooksTestSupport;
 import com.kingsrook.qbits.webhooks.WebhooksTestSupport.Receiver;
 import com.kingsrook.qbits.webhooks.delivery.DeliveryEngine;
@@ -29,11 +34,14 @@ import com.kingsrook.qbits.webhooks.metadata.WebhooksTables;
 import com.kingsrook.qbits.webhooks.permissions.WebhooksPermissions;
 import com.kingsrook.qbits.webhooks.support.SystemWrites;
 import com.kingsrook.qqq.backend.core.context.QContext;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -133,6 +141,31 @@ class WebhooksHomeTest
 
 
 
+   /*******************************************************************************
+    ** The KPI drill links open the delivery or attempt table with the matching
+    ** filter, as URL-encoded JSON.
+    *******************************************************************************/
+   @Test
+   void kpiDrillLinksAreFilteredJson() throws Exception
+   {
+      String html = home();
+      Matcher matcher = Pattern.compile("href=\"(/admin/webhooks/([a-z_]+)\\?filter=([^\"]+))\"").matcher(html);
+      Map<String, QQueryFilter> byTable = new HashMap<>();
+      int links = 0;
+      while(matcher.find())
+      {
+         links++;
+         QQueryFilter filter = JsonUtils.toObject(URLDecoder.decode(matcher.group(3), StandardCharsets.UTF_8), QQueryFilter.class);
+         byTable.put(matcher.group(2) + ":" + filter.getCriteria().getFirst().getFieldName(), filter);
+      }
+      assertThat(links).isEqualTo(4);
+      assertThat(byTable).containsKeys(WebhooksTables.DELIVERY + ":deliveredDate", WebhooksTables.ATTEMPT + ":startDate", WebhooksTables.DELIVERY + ":status");
+      assertThat(byTable.get(WebhooksTables.DELIVERY + ":status").getCriteria()).hasSize(1);
+      assertThat(byTable.get(WebhooksTables.ATTEMPT + ":startDate").getCriteria().get(1).getOperator()).isEqualTo(QCriteriaOperator.NOT_IN);
+   }
+
+
+
    /*******************************************************************************
     ** The secret value: admins only, behind a toggle; references not shown.
     *******************************************************************************/

src/test/java/com/kingsrook/qbits/webhooks/health/WebhooksHealthWidgetTest.java

@@ -23,18 +23,20 @@ package com.kingsrook.qbits.webhooks.health;
 
 import java.net.URLDecoder;
 import java.nio.charset.StandardCharsets;
-import java.util.Base64;
 import java.util.List;
 import com.kingsrook.qbits.webhooks.WebhooksTestSupport;
 import com.kingsrook.qbits.webhooks.app.WebhooksApp;
 import com.kingsrook.qbits.webhooks.support.SystemWrites;
 import com.kingsrook.qqq.backend.core.context.QContext;
 import com.kingsrook.qqq.backend.core.health.HealthMetaDataProvider;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
+import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
 import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.StatisticsData;
 import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
 import com.kingsrook.qqq.backend.core.model.data.QRecord;
 import com.kingsrook.qqq.backend.core.model.session.QSession;
+import com.kingsrook.qqq.backend.core.utils.JsonUtils;
 import org.junit.jupiter.api.Test;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -55,8 +57,15 @@ class WebhooksHealthWidgetTest
       StatisticsData data = render();
       assertThat(data.getCount()).isEqualTo("No checks have run yet");
       assertThat(data.getCountURL()).startsWith("/app/health_check?filter=");
-      String filter = new String(Base64.getDecoder().decode(URLDecoder.decode(data.getCountURL().split("filter=")[1], StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
-      assertThat(filter).contains("component", "webhooks");
+      ///////////////////////////////////////////////////////////////////////////////
+      // plain URL-encoded JSON (the form every qbit uses), not base64              //
+      ///////////////////////////////////////////////////////////////////////////////
+      String filter = URLDecoder.decode(data.getCountURL().split("filter=")[1], StandardCharsets.UTF_8);
+      QQueryFilter decoded = JsonUtils.toObject(filter, QQueryFilter.class);
+      assertThat(decoded.getCriteria()).hasSize(1);
+      assertThat(decoded.getCriteria().getFirst().getFieldName()).isEqualTo("component");
+      assertThat(decoded.getCriteria().getFirst().getOperator()).isEqualTo(QCriteriaOperator.EQUALS);
+      assertThat(decoded.getCriteria().getFirst().getValues()).containsExactly("webhooks");
       new HealthMetaDataProvider().defineAll(instance, WebhooksTestSupport.MEMORY, null);
       QContext.init(instance, new QSession().withPermissions(List.of("health.read")));
       assertThat(render().getCount()).isEqualTo("No checks have run yet");