Compare commits

..
7 Commits
13 changed files with 97 additions and 234 deletions

No files matched your search

+61 -10
View File
@@ -1,14 +1,65 @@
version: 2.1
setup: true
orbs:
qqq-orb: kingsrook/qqq-orb@0.8
qqq-orb: kingsrook/qqq-orb@0.6.9
commands:
configure-package-install:
steps:
- run:
name: Keep package installation unattended
command: |
# These build machines are discarded after the job.
sudo install -d /etc/needrestart/conf.d
printf '%s\n' '$nrconf{restart} = "l";' | sudo tee /etc/needrestart/conf.d/99-qqq-ci.conf >/dev/null
workflows:
setup:
test_only:
jobs:
- qqq-orb/generate_pipeline:
orb_version: '0.8'
filters:
branches:
ignore: automation/qbit-rc-pending
tags:
only: /^v\d+\.\d+\.\d+$/
- qqq-orb/mvn_test_only:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
filters:
branches:
ignore: /(develop|release/.*|integration.*)/
tags:
only: []
publish_snapshot:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: snapshot
filters:
branches:
only:
- develop
publish_release_candidate:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: release_candidate
filters:
branches:
only:
- /release\/.*/
publish_release:
jobs:
- qqq-orb/mvn_publish:
pre-steps:
- configure-package-install
context: [ qqq-maven-registry-credentials ]
branch_type: release
filters:
branches:
ignore: /.*/
tags:
only: [ /^v\d+\.\d+\.\d+$/ ]
-7
View File
@@ -1,7 +0,0 @@
name: build
on:
push:
jobs:
maven:
uses: self:KofTwentyTwo/ci-workflows/.gitea/workflows/maven.yaml@v1
secrets: inherit
-23
View File
@@ -1,23 +0,0 @@
name: CodeQL
'on':
push:
branches:
- main
- develop
- release/**
- hotfix/**
pull_request: {}
schedule:
- cron: 41 16 * * 0
workflow_dispatch: {}
permissions:
contents: read
actions: read
security-events: write
packages: read
concurrency:
group: codeql-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
analyze:
uses: QRun-IO/qqq-orb/.github/workflows/qbit-codeql.yml@c288a8864fe9054f68410e1d78f5863aea4c4d22
+13 -62
View File
@@ -1,74 +1,25 @@
# Cut the next release candidate (QRun-IO/qqq docs/QRUN-QBIT-REPO-STANDARD.md, C6):
# run this workflow by hand, or merge a develop PR labelled `release:rc`. It merges develop into
# the release line and the release/* push runs CircleCI publish_release_candidate.
name: Release candidate
'on':
on:
workflow_dispatch:
inputs:
dry_run:
description: Plan a cut without changing release state
description: Print the plan without merging or creating branches
type: boolean
default: false
finalize:
description: Retry finalization of an existing cut; do not cut another RC
type: boolean
default: false
recover:
description: Recover the same published RC after a deployment timeout; never
reupload
type: boolean
default: false
circle_build:
description: Original Circle publisher build number (recovery only)
type: string
default: ''
merge_sha:
description: Existing cut merge_sha (finalize retry only)
type: string
default: ''
bump_sha:
description: Existing cut bump_sha (finalize retry only)
type: string
default: ''
branch:
description: Existing cut branch (finalize retry only)
type: string
default: ''
version:
description: Existing cut version (finalize retry only)
type: string
default: ''
pull_request:
types:
- closed
branches:
- develop
repository_dispatch:
types:
- qbit-rc-published
types: [closed]
branches: [develop]
permissions:
contents: write
actions: read
checks: read
jobs:
release-rc:
if: github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch'
|| (github.event_name == 'pull_request' && github.event.pull_request.merged
== true && github.event.pull_request.base.ref == 'develop' && contains(github.event.pull_request.labels.*.name,
'release:rc'))
uses: QRun-IO/qqq-orb/.github/workflows/qbit-release-rc.yml@cc30f599e63c8e41629c99d23395bac1adb8bc44
if: github.event_name == 'workflow_dispatch' || (github.event.pull_request.merged == true && contains(github.event.pull_request.labels.*.name, 'release:rc'))
uses: QRun-IO/.github/.github/workflows/qbit-release-rc.yml@main
with:
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run || false
}}
finalize: ${{ github.event_name == 'repository_dispatch' || (github.event_name
== 'workflow_dispatch' && inputs.finalize) }}
recover: ${{ github.event_name == 'workflow_dispatch' && inputs.recover || false
}}
circle_build: ${{ github.event_name == 'workflow_dispatch' && inputs.circle_build
|| '' }}
merge_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.merge_sha
|| inputs.merge_sha || '' }}
bump_sha: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.bump_sha
|| inputs.bump_sha || '' }}
branch: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.branch
|| inputs.branch || '' }}
version: ${{ github.event_name == 'repository_dispatch' && github.event.client_payload.version
|| inputs.version || '' }}
secrets: inherit
dry_run: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
-16
View File
@@ -1,16 +0,0 @@
profile: qbit
orb_version: '0.8'
default_branch: develop
java_version: '21'
browsers: true
api_checks: false
docs_only: false
release_on_main: false
release_tag_policy: stable
test_branch_policy: non-release-no-integration
publish:
snapshot: true
release_candidate: true
release: true
asciidoc: false
release_completion: true
-6
View File
@@ -6,12 +6,6 @@ All notable changes are documented here, following [Keep a Changelog](https://ke
### Fixed
- Upload-job status transitions no longer use `status` as the table's record-version field.
QQQ 4.1.0-RC.26 advances a configured version field on every update (a STRING version
becomes a random UUID), which overwrote upload status. The guarded transition now re-reads
status inside the existing cross-pod account-lock transaction. The upload-job table no longer
declares a record-version field; no schema or public API change.
- Endpoint pickers for saved views, saved reports and saved bulk-load profiles are dropped when
the host has not registered those possible-value sources, so hosts without them validate.
+5 -6
View File
@@ -22,8 +22,7 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<properties>
<qqq.version>4.1.0-RC.26</qqq.version> <!-- feeds qbit-build-parent's qqq-bom-pom import -->
<revision>0.1.0-SNAPSHOT</revision>
<revision>0.1.0-RC.4</revision>
<maven.compiler.source>21</maven.compiler.source>
<maven.compiler.target>21</maven.compiler.target>
<jacoco.version>0.8.15</jacoco.version>
@@ -34,7 +33,7 @@
<parent>
<groupId>com.kingsrook</groupId>
<artifactId>qbit-build-parent</artifactId>
<version>2.2.1</version>
<version>2.0.0</version>
</parent>
<groupId>com.kingsrook.qbits</groupId>
@@ -82,7 +81,7 @@
<dependency>
<groupId>com.kingsrook.qbits</groupId>
<artifactId>qbit-object-storage</artifactId>
<version>0.1.0-RC.10</version>
<version>0.1.0-RC.2</version>
</dependency>
<!-- MINA's Ed25519 key decoder and signer. -->
<dependency>
@@ -118,7 +117,7 @@
<dependency>
<groupId>com.kingsrook.qqq</groupId>
<artifactId>qqq-frontend-next</artifactId>
<version>1.0.0-RC.38</version>
<version>1.0.0-RC.14</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -240,7 +239,7 @@
</activation>
<properties>
<!-- QQQ 4.1.0-RC.7: the first released core with the UiApiInvoker in-process sessions, streams and multipart (QRun-IO/qqq#1170). Never a SNAPSHOT (guardrail G7). -->
<qqq.snapshot.version>4.1.0-RC.26</qqq.snapshot.version>
<qqq.snapshot.version>4.1.0-RC.7</qqq.snapshot.version>
</properties>
<dependencyManagement>
<dependencies>
@@ -108,13 +108,13 @@ public class FileGatewayHome extends AbstractWidgetRenderer
long value = "transfers".equals(key) ? rows.size() : rows.stream()
.filter(row -> ("in".equals(key) ? "UPLOAD" : "DOWNLOAD").equals(row.getValueString("operation")))
.mapToLong(row -> number(row, "bytes")).sum();
return (new StatisticsData("transfers".equals(key) ? value : Math.round(value / 10485.76) / 100.0, null, null).withIncreaseIsGood(true));
return (new StatisticsData("transfers".equals(key) ? value : Math.round(value / 10485.76) / 100.0, null, null));
}
if("locked".equals(key))
{
long count = query(FileGatewayTables.ACCOUNT, new QQueryFilter()).stream().filter(row -> "LOCKED".equals(row.getValueString("status"))
|| (row.getValueInstant("lockedUntil") != null && row.getValueInstant("lockedUntil").isAfter(now))).count();
return (new StatisticsData(count, null, null).withIncreaseIsGood(false));
return (new StatisticsData(count, null, null));
}
if(List.of("failures", "recent").contains(key))
{
@@ -122,26 +122,24 @@ public class FileGatewayHome extends AbstractWidgetRenderer
.withCriteria(new QFilterCriteria("status", QCriteriaOperator.IN, "FAILED", "PARTIAL", "REJECTED")));
if("failures".equals(key))
{
return (new StatisticsData(rows.size(), null, null).withIncreaseIsGood(false));
return (new StatisticsData(rows.size(), null, null));
}
rows.sort(java.util.Comparator.comparing((QRecord row) -> row.getValueInstant("finishedAt")).reversed());
return (table("Recent failures", FileGatewayTables.UPLOAD_JOB, rows.stream().limit(20).toList(),
List.of("fileName", "status", "errorMessage", "finishedAt"), List.of("File", "Status", "Message", "Finished"), "No failed transfers in the last 7 days."));
List.of("fileName", "status", "errorMessage", "finishedAt"), List.of("File", "Status", "Message", "Finished")));
}
if(List.of("sessions", "active").contains(key))
{
var rows = query(FileGatewayTables.CONNECTION_LOG, since("lastSeenAt", now.minusSeconds(180))
.withCriteria(new QFilterCriteria("disconnectedAt", QCriteriaOperator.IS_BLANK))
.withCriteria(new QFilterCriteria("outcome", QCriteriaOperator.EQUALS, "CONNECTED")));
return ("sessions".equals(key) ? new StatisticsData(rows.size(), null, null).withIncreaseIsGood(true) : table("Active sessions", FileGatewayTables.CONNECTION_LOG, rows,
List.of("username", "remoteIp", "connectedAt", "operationCount"), List.of("Username", "IP", "Connected", "Operations"),
"No one is connected right now (no session has been seen in the last 3 minutes)."));
return ("sessions".equals(key) ? new StatisticsData(rows.size(), null, null) : table("Active sessions", FileGatewayTables.CONNECTION_LOG, rows,
List.of("username", "remoteIp", "connectedAt", "operationCount"), List.of("Username", "IP", "Connected", "Operations")));
}
if("servers".equals(key))
{
return (table("Server", FileGatewayTables.SERVER, query(FileGatewayTables.SERVER, since("lastSeenAt", now.minusSeconds(180))),
List.of("podName", "hostKeyFingerprint", "port", "lastSeenAt"), List.of("Pod", "Host key fingerprint", "Port", "Last seen"),
"No gateway server has reported in during the last 3 minutes."));
List.of("podName", "hostKeyFingerprint", "port", "lastSeenAt"), List.of("Pod", "Host key fingerprint", "Port", "Last seen")));
}
Map<Integer, QRecord> partners = new LinkedHashMap<>();
for(QRecord account : query(FileGatewayTables.ACCOUNT, new QQueryFilter()))
@@ -159,8 +157,7 @@ public class FileGatewayHome extends AbstractWidgetRenderer
}
List<QRecord> rows = partners.values().stream().filter(row -> number(row, "transfers") > 0)
.sorted(java.util.Comparator.comparingLong((QRecord row) -> number(row, "transfers")).reversed()).limit(10).toList();
return (table("Top partners (7 days)", FileGatewayTables.ACCOUNT, rows, List.of("username", "transfers", "mb"), List.of("Partner", "Transfers", "MB"),
"No partner has transferred a file in the last 7 days."));
return (table("Top partners (7 days)", FileGatewayTables.ACCOUNT, rows, List.of("username", "transfers", "mb"), List.of("Partner", "Transfers", "MB")));
}
@@ -168,7 +165,7 @@ public class FileGatewayHome extends AbstractWidgetRenderer
/*******************************************************************************
** Typed text blocks carry record links; cells never contain markup.
*******************************************************************************/
private TableData table(String label, String table, List<QRecord> records, List<String> fields, List<String> labels, String noRowsText)
private TableData table(String label, String table, List<QRecord> records, List<String> fields, List<String> labels)
{
List<TableData.Column> columns = new ArrayList<>();
for(int i = 0; i < fields.size(); i++)
@@ -188,7 +185,7 @@ public class FileGatewayHome extends AbstractWidgetRenderer
.withLink("/record/" + table + "/" + record.getValue("id")));
rows.add(row);
}
return (new TableData(null, columns, rows).withRowsPerPage(10).withHidePaginationDropdown(true).withNoRowsFoundHTML(noRowsText));
return (new TableData(null, columns, rows).withRowsPerPage(10).withHidePaginationDropdown(true));
}
@@ -297,7 +297,7 @@ public final class FileGatewayTables
*******************************************************************************/
private static QTableMetaData upload_job(FileGatewayQBitConfig config)
{
QTableMetaData table = base(config, UPLOAD_JOB, "Upload Jobs")
QTableMetaData table = base(config, UPLOAD_JOB, "Upload Jobs").withRecordVersionField("status")
.withRecordLabelFormatAndFields("%s", "id")
.withField(field("accountId", QFieldType.INTEGER).withPossibleValueSourceName(ACCOUNT))
.withField(field("aliasId", QFieldType.INTEGER).withPossibleValueSourceName(ALIAS))
@@ -22,19 +22,14 @@ package com.kingsrook.qbits.filegateway.permissions;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qqq.backend.core.actions.permissions.AvailablePermission;
import com.kingsrook.qqq.backend.core.actions.permissions.CustomPermissionChecker;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionSubType;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.AbstractActionInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithName;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.MetaDataWithPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
@@ -65,42 +60,4 @@ public class FileGatewayPermissionChecker implements CustomPermissionChecker
throw (new QPermissionDeniedException("Missing file gateway permission"));
}
}
/*******************************************************************************
** This checker gates on the three gateway permission names, not on per-object
** names, so it tells the catalog (getAllAvailablePermissions) those names.
*******************************************************************************/
@Override
public boolean handlesBuildAvailablePermission()
{
return (true);
}
/*******************************************************************************
** Collapse every gateway object onto file_gateway.view, .admin or .connect.
** The object name and type are fixed so the catalog (a set) lists each name
** once: reads and the app need view, writes and processes need admin, and the
** app also stands for SFTP sign-in (connect), which the gateway grants to
** sessions rather than checking on an object.
*******************************************************************************/
@Override
public AvailablePermission buildAvailablePermission(QPermissionRules rules, PermissionSubType permissionSubType, String baseName, MetaDataWithName metaDataWithName, String objectType)
{
AvailablePermission base = buildBaseAvailablePermission(rules, permissionSubType, metaDataWithName, objectType);
if(base == null)
{
return (null);
}
boolean write = "WRITE".equals(base.getPermissionType()) || "Process".equals(objectType);
String name = write ? FileGatewayPermissions.ADMIN : FileGatewayPermissions.VIEW;
if(metaDataWithName instanceof QAppMetaData)
{
name = FileGatewayPermissions.CONNECT;
}
return (base.withName(name).withObjectName("File Gateway").withObjectType("App").withPermissionType(name.substring(name.indexOf('.') + 1)));
}
}
@@ -24,7 +24,7 @@ package com.kingsrook.qbits.filegateway.sftp;
import java.io.Serializable;
import java.time.Instant;
import java.util.List;
import java.util.Objects;
import java.util.Map;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qqq.backend.core.actions.QBackendTransaction;
import com.kingsrook.qqq.backend.core.actions.permissions.CorePermissionEnforcer;
@@ -244,23 +244,18 @@ public class AuditWriter
/*******************************************************************************
** Guarded status transition. Callers hold withAccountLock, so the status re-read in
** that transaction is serialized across pods; status is compared here rather than
** through a core record-version field, because core advances version fields on
** every update (a STRING version becomes a random UUID).
** Optimistic status transition, called under withAccountLock because core checks
** expected versions before SQL execution rather than in the SQL WHERE clause.
*******************************************************************************/
public boolean transitionUpload(QRecord changes, String expectedStatus) throws QException
{
if(transaction.get() == null)
{
throw (new QException("Upload transitions require the account lock transaction"));
}
var rows = query(FileGatewayTables.UPLOAD_JOB, new QQueryFilter(new QFilterCriteria("id", QCriteriaOperator.EQUALS, changes.getValue("id"))));
if(rows.isEmpty() || !Objects.equals(expectedStatus, rows.getFirst().getValueString("status")))
QRecord result = system(() -> new UpdateAction().execute(new UpdateInput(FileGatewayTables.UPLOAD_JOB)
.withRecords(List.of(changes)).withTransaction(transaction.get())
.withExpectedVersions(Map.of(changes.getValue("id"), expectedStatus))).getRecords().getFirst());
if(result.getErrors() != null && !result.getErrors().isEmpty())
{
return (false);
}
update(FileGatewayTables.UPLOAD_JOB, changes);
return (true);
}
@@ -22,8 +22,6 @@ package com.kingsrook.qbits.filegateway;
import java.util.List;
import com.kingsrook.qbits.filegateway.metadata.FileGatewayTables;
import com.kingsrook.qbits.filegateway.permissions.FileGatewayPermissions;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.ClientChannelsMetaData;
import org.junit.jupiter.api.Test;
@@ -75,21 +73,6 @@ class FileGatewayQBitProducerTest
/*******************************************************************************
** The three gateway permission names reach QQQ's catalog, so identity
** policies can grant them.
*******************************************************************************/
@Test
void permissionsAreInTheAvailableCatalog() throws Exception
{
var instance = new QInstance();
var config = new FileGatewayQBitConfig().withBackendName("host-db");
new FileGatewayQBitProducer().withQBitConfig(config).withNamespace("test").produce(instance).addSelfToInstance(instance);
assertThat(PermissionsHelper.getAllAvailablePermissionNames(instance)).containsAll(FileGatewayPermissions.ALL);
}
/*******************************************************************************
** Package scanning never enables an unconfigured producer.
*******************************************************************************/
@@ -274,24 +274,6 @@ class PhaseEIntegrationTest
assertThat(((Number) ((StatisticsData) home.data("out", now)).getCount()).doubleValue()).isEqualTo(1);
assertThat(((Number) ((StatisticsData) home.data("failures", now)).getCount()).intValue()).isEqualTo(1);
assertThat(((TableData) home.data("recent", now)).getRows()).hasSize(1);
///////////////////////////////////////////////////////////////////////////////
// direction: failures and locks are bad when they grow; volume is not bad //
///////////////////////////////////////////////////////////////////////////////
assertThat(((StatisticsData) home.data("failures", now)).getIncreaseIsGood()).as("failures").isFalse();
assertThat(((StatisticsData) home.data("locked", now)).getIncreaseIsGood()).as("locked").isFalse();
for(String key : List.of("transfers", "in", "out", "sessions"))
{
assertThat(((StatisticsData) home.data(key, now)).getIncreaseIsGood()).as(key).isTrue();
}
///////////////////////////////////////////////////////////////////////////////
// every home table says why it is empty //
///////////////////////////////////////////////////////////////////////////////
for(String key : List.of("recent", "partners", "active", "servers"))
{
assertThat(((TableData) home.data(key, now)).getNoRowsFoundHTML()).as(key).isNotBlank();
}
assertThat(((TableData) home.data("active", now)).getRows()).isEmpty();
assertThat(((TableData) home.data("active", now)).getNoRowsFoundHTML()).contains("No one is connected");
//////////////////////////////////////////////////////////////////
// readable at all widths: home tables declare no column widths //
//////////////////////////////////////////////////////////////////