From 77d8d90ebb769c5cf098902434b8ef49bdebef09 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 4 Nov 2025 16:54:27 -0600 Subject: [PATCH 1/5] feat(backendActivities): Update standard process traces to optionally also store backend activity stats That is: number of queries, inserts, etc, per table, used within a process. This feature does depend on the application server having the QueryStatManager started, and adding this package's StandardProcessTraceQueryStatConsumer to QueryStatManager's queryStatConsumers list, and this qbit being configured with includeBackendActivityStats = true --- ...ndActivityStatsActionMetaDataProducer.java | 62 ++++ ...eBackendActivityStatsMetaDataProducer.java | 93 +++++ ...nBackendActivityStatsMetaDataProducer.java | 71 ++++ ...ndActivityStatsWidgetMetaDataProducer.java | 72 ++++ .../model/ProcessTrace.java | 22 +- .../ProcessTraceBackendActivityStats.java | 349 ++++++++++++++++++ ...rocessTraceBackendActivityStatsAction.java | 132 +++++++ ...ocessTraceBackendActivityStatsManager.java | 298 +++++++++++++++ ...StandardProcessTraceQueryStatConsumer.java | 67 ++++ .../utils/StandardProcessTracer.java | 56 ++- 10 files changed, 1215 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceBackendActivityStatsActionMetaDataProducer.java create mode 100644 src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceBackendActivityStatsMetaDataProducer.java create mode 100644 src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceJoinBackendActivityStatsMetaDataProducer.java create mode 100644 src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceJoinBackendActivityStatsWidgetMetaDataProducer.java create mode 100644 src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTraceBackendActivityStats.java create mode 100644 src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTraceBackendActivityStatsAction.java create mode 100644 src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/ProcessTraceBackendActivityStatsManager.java create mode 100644 src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTraceQueryStatConsumer.java diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceBackendActivityStatsActionMetaDataProducer.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceBackendActivityStatsActionMetaDataProducer.java new file mode 100644 index 0000000..9b39805 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceBackendActivityStatsActionMetaDataProducer.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.metadata; + + +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceBackendActivityStatsAction; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValueSource; + + +/******************************************************************************* + * Meta Data Producer for ProcessTraceBackendActivityStatsAction + * Note: not done via annotations, so it can be disabled via config + *******************************************************************************/ +public class ProcessTraceBackendActivityStatsActionMetaDataProducer extends MetaDataProducer +{ + public static final String NAME = ProcessTraceBackendActivityStatsAction.NAME; + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (new ProcessTraceBackendActivityStatsMetaDataProducer().isEnabled()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QPossibleValueSource produce(QInstance qInstance) throws QException + { + return (QPossibleValueSource.newForEnum(NAME, ProcessTraceBackendActivityStatsAction.values())); + } + +} diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceBackendActivityStatsMetaDataProducer.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceBackendActivityStatsMetaDataProducer.java new file mode 100644 index 0000000..8c6021b --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceBackendActivityStatsMetaDataProducer.java @@ -0,0 +1,93 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.metadata; + + +import java.util.List; +import com.kingsrook.qbits.standardprocesstrace.StandardProcessTraceQBitConfig; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTrace; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceBackendActivityStats; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProductionContext; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.ExposedJoin; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.SectionFactory; + + +/******************************************************************************* + * Meta Data Producer for ProcessTraceBackendActivityStats + * Note: not done via annotations, so it can be disabled via config + *******************************************************************************/ +public class ProcessTraceBackendActivityStatsMetaDataProducer extends MetaDataProducer +{ + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean isEnabled() + { + QBitConfig qBitConfig = QBitProductionContext.peekQBitConfig(); + if(qBitConfig instanceof StandardProcessTraceQBitConfig config) + { + if(!config.getIncludeBackendActivityStats()) + { + return (false); + } + } + + return (true); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QTableMetaData produce(QInstance qInstance) throws QException + { + QTableMetaData table = new QTableMetaData() + .withName(ProcessTraceBackendActivityStats.TABLE_NAME) + .withIcon(new QIcon().withName("query_stats")) + .withRecordLabelFormat("%s %s %s %s") + .withRecordLabelFields("processTraceId", "qqqBackendId", "qqqTableId", "actionName") + .withPrimaryKeyField("id") + .withFieldsFromEntity(ProcessTraceBackendActivityStats.class) + .withSection(SectionFactory.defaultT1("id", "processTraceId", "qqqBackendId", "qqqTableId", "actionName")) + .withSection(SectionFactory.defaultT2("callCount", "recordCount", "runtimeMillis")) + .withExposedJoin(new ExposedJoin().withLabel("Process Trace").withJoinPath(List.of(ProcessTraceJoinBackendActivityStatsMetaDataProducer.NAME)).withJoinTable(ProcessTrace.TABLE_NAME)); + + table.withoutCapabilities(Capability.allWriteCapabilities()); + + QInstanceEnricher.setInferredFieldBackendNames(table); + + return (table); + } + +} diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceJoinBackendActivityStatsMetaDataProducer.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceJoinBackendActivityStatsMetaDataProducer.java new file mode 100644 index 0000000..288c09f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceJoinBackendActivityStatsMetaDataProducer.java @@ -0,0 +1,71 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.metadata; + + +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTrace; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceBackendActivityStats; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinOn; +import com.kingsrook.qqq.backend.core.model.metadata.joins.JoinType; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; + + +/******************************************************************************* + * Meta Data Producer for ProcessTraceJoinBackendActivityStats + * Note: not done via annotations, so it can be disabled via config + *******************************************************************************/ +public class ProcessTraceJoinBackendActivityStatsMetaDataProducer extends MetaDataProducer +{ + public static final String NAME = "ProcessTraceJoinBackendActivityStats"; + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (new ProcessTraceBackendActivityStatsMetaDataProducer().isEnabled()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QJoinMetaData produce(QInstance qInstance) throws QException + { + return (new QJoinMetaData() + .withName(NAME) + .withLeftTable(ProcessTrace.TABLE_NAME) + .withRightTable(ProcessTraceBackendActivityStats.TABLE_NAME) + .withType(JoinType.ONE_TO_MANY) + .withJoinOn(new JoinOn("id", "processTraceId")) + ); + } + +} diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceJoinBackendActivityStatsWidgetMetaDataProducer.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceJoinBackendActivityStatsWidgetMetaDataProducer.java new file mode 100644 index 0000000..ffc00d1 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/metadata/ProcessTraceJoinBackendActivityStatsWidgetMetaDataProducer.java @@ -0,0 +1,72 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.metadata; + + +import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.ChildRecordListRenderer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducer; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.joins.QJoinMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitProductionContext; + + +/******************************************************************************* + * Meta Data Producer for ProcessTraceJoinBackendActivityStats + * Note: not done via annotations, so it can be disabled via config + *******************************************************************************/ +public class ProcessTraceJoinBackendActivityStatsWidgetMetaDataProducer extends MetaDataProducer +{ + public static final String NAME = "ProcessTraceJoinBackendActivityStatsWidget"; + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public boolean isEnabled() + { + return (new ProcessTraceBackendActivityStatsMetaDataProducer().isEnabled()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public QWidgetMetaData produce(QInstance qInstance) throws QException + { + MetaDataProducerMultiOutput metaDataProducerMultiOutput = QBitProductionContext.peekMetaDataProducerMultiOutput(); + QJoinMetaData joinMetaData = metaDataProducerMultiOutput.get(QJoinMetaData.class, ProcessTraceJoinBackendActivityStatsMetaDataProducer.NAME); + + return ChildRecordListRenderer.widgetMetaDataBuilder(joinMetaData) + .withName(NAME) + .withLabel("Backend Activity Stats") + .withMaxRows(250) + .getWidgetMetaData(); + } + +} diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTrace.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTrace.java index 49c7e11..e904b00 100644 --- a/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTrace.java +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTrace.java @@ -24,6 +24,9 @@ package com.kingsrook.qbits.standardprocesstrace.model; import java.time.Instant; import java.util.List; +import com.kingsrook.qbits.standardprocesstrace.metadata.ProcessTraceBackendActivityStatsMetaDataProducer; +import com.kingsrook.qbits.standardprocesstrace.metadata.ProcessTraceJoinBackendActivityStatsMetaDataProducer; +import com.kingsrook.qbits.standardprocesstrace.metadata.ProcessTraceJoinBackendActivityStatsWidgetMetaDataProducer; import com.kingsrook.qbits.standardprocesstrace.metadata.ProcessTraceJoinKeyRecordQQQTableMetaDataProducer; import com.kingsrook.qbits.standardprocesstrace.metadata.ProcessTraceJoinQQQProcessMetaDataProducer; import com.kingsrook.qbits.standardprocesstrace.utils.ProcessTraceTableCustomizer; @@ -66,7 +69,7 @@ import com.kingsrook.qqq.backend.core.model.tables.QQQTable; childTableEntityClass = ProcessTraceSummaryLine.class, joinFieldName = "processTraceSummaryLineId", childJoin = @ChildJoin(enabled = true), - childRecordListWidget = @ChildRecordListWidget(label = "Summary Lines", enabled = true, maxRows = 250)) + childRecordListWidget = @ChildRecordListWidget(label = "Summary Lines", enabled = true, maxRows = 250)), } ) public class ProcessTrace extends QRecordEntity @@ -87,7 +90,7 @@ public class ProcessTrace extends QRecordEntity @Override public QTableMetaData customizeMetaData(QInstance qInstance, QTableMetaData table) throws QException { - String childJoinName = QJoinMetaData.makeInferredJoinName(TABLE_NAME, ProcessTraceSummaryLine.TABLE_NAME); + String summaryLinesChildJoinName = QJoinMetaData.makeInferredJoinName(TABLE_NAME, ProcessTraceSummaryLine.TABLE_NAME); table .withUniqueKey(new UniqueKey("processUUID")) @@ -96,14 +99,25 @@ public class ProcessTrace extends QRecordEntity .withRecordLabelFields("qqqProcessId", "processUUID") .withSection(new QFieldSection("identity", new QIcon().withName("badge"), Tier.T1, List.of("id", "qqqProcessId", "processUUID", "userId"))) .withSection(new QFieldSection("data", new QIcon().withName("text_snippet"), Tier.T2, List.of("startTimestamp", "endTimestamp", "runtimeMillis", "keyRecordQqqTableId", "keyRecordId", "recordCount", "exceptionMessage"))) - .withSection(new QFieldSection("summaryLines", new QIcon().withName("horizontal_rule"), Tier.T2).withWidgetName(childJoinName)) - .withExposedJoin(new ExposedJoin().withLabel("Summary Lines").withJoinPath(List.of(childJoinName)).withJoinTable(ProcessTraceSummaryLine.TABLE_NAME)) + .withSection(new QFieldSection("summaryLines", new QIcon().withName("horizontal_rule"), Tier.T2).withWidgetName(summaryLinesChildJoinName)) + .withExposedJoin(new ExposedJoin().withLabel("Summary Lines").withJoinPath(List.of(summaryLinesChildJoinName)).withJoinTable(ProcessTraceSummaryLine.TABLE_NAME)) .withExposedJoin(new ExposedJoin().withLabel("Key Record Table").withJoinPath(List.of(ProcessTraceJoinKeyRecordQQQTableMetaDataProducer.NAME)).withJoinTable(QQQTable.TABLE_NAME)) .withExposedJoin(new ExposedJoin().withLabel("Process").withJoinPath(List.of(ProcessTraceJoinQQQProcessMetaDataProducer.NAME)).withJoinTable(QQQProcess.TABLE_NAME)); table.withCustomizer(TableCustomizers.POST_QUERY_RECORD, new QCodeReference(ProcessTraceTableCustomizer.class)); table.getField("keyRecordId").withFieldAdornment(new FieldAdornment(AdornmentType.LINK).withValue(AdornmentType.LinkValues.TO_RECORD_FROM_TABLE_DYNAMIC, true)); + if(new ProcessTraceBackendActivityStatsMetaDataProducer().isEnabled()) + { + table.getSections().add(new QFieldSection("backendActivityStats", new QIcon().withName("query_stats"), Tier.T2) + .withWidgetName(ProcessTraceJoinBackendActivityStatsWidgetMetaDataProducer.NAME)); + + table.withExposedJoin(new ExposedJoin() + .withLabel("Backend Activity Stats") + .withJoinPath(List.of(ProcessTraceJoinBackendActivityStatsMetaDataProducer.NAME)) + .withJoinTable(ProcessTraceBackendActivityStats.TABLE_NAME)); + } + return (table); } } diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTraceBackendActivityStats.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTraceBackendActivityStats.java new file mode 100644 index 0000000..c74263f --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTraceBackendActivityStats.java @@ -0,0 +1,349 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.model; + + +import com.kingsrook.qqq.backend.core.model.backends.QQQBackend; +import com.kingsrook.qqq.backend.core.model.data.QField; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.data.QRecordEntity; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.tables.QQQTable; + + +/******************************************************************************* + * QRecord Entity for ProcessTraceBackendActivityStats table + * Note: not using meta-data producing annotations, so it can be disabled via config + *******************************************************************************/ +public class ProcessTraceBackendActivityStats extends QRecordEntity +{ + public static final String TABLE_NAME = "processTraceBackendActivityStats"; + + @QField(isEditable = false, isPrimaryKey = true) + private Long id; + + @QField(possibleValueSourceName = ProcessTrace.TABLE_NAME) + private Long processTraceId; + + @QField(label = "Backend", possibleValueSourceName = QQQBackend.TABLE_NAME) + private Integer qqqBackendId; + + @QField(label = "Table", possibleValueSourceName = QQQTable.TABLE_NAME) + private Integer qqqTableId; + + @QField(label = "Action", possibleValueSourceName = ProcessTraceBackendActivityStatsAction.NAME) + private String actionName; + + @QField(displayFormat = DisplayFormat.COMMAS) + private Integer callCount; + + @QField(displayFormat = DisplayFormat.COMMAS) + private Integer recordCount; + + @QField(displayFormat = DisplayFormat.COMMAS) + private Integer runtimeMillis; + + + + /******************************************************************************* + ** Default constructor + *******************************************************************************/ + public ProcessTraceBackendActivityStats() + { + } + + + + /******************************************************************************* + ** Constructor that takes a QRecord + *******************************************************************************/ + public ProcessTraceBackendActivityStats(QRecord record) + { + populateFromQRecord(record); + } + + + + /******************************************************************************* + * Getter for id + * @see #withId(Long) + *******************************************************************************/ + public Long getId() + { + return (this.id); + } + + + + /******************************************************************************* + * Setter for id + * @see #withId(Long) + *******************************************************************************/ + public void setId(Long id) + { + this.id = id; + } + + + + /******************************************************************************* + * Fluent setter for id + *******************************************************************************/ + public ProcessTraceBackendActivityStats withId(Long id) + { + this.id = id; + return (this); + } + + + + /******************************************************************************* + * Getter for processTraceId + * @see #withProcessTraceId(Long) + *******************************************************************************/ + public Long getProcessTraceId() + { + return (this.processTraceId); + } + + + + /******************************************************************************* + * Setter for processTraceId + * @see #withProcessTraceId(Long) + *******************************************************************************/ + public void setProcessTraceId(Long processTraceId) + { + this.processTraceId = processTraceId; + } + + + + /******************************************************************************* + * Fluent setter for processTraceId + *******************************************************************************/ + public ProcessTraceBackendActivityStats withProcessTraceId(Long processTraceId) + { + this.processTraceId = processTraceId; + return (this); + } + + + + + /******************************************************************************* + * Getter for actionName + * @see #withActionName(String) + *******************************************************************************/ + public String getActionName() + { + return (this.actionName); + } + + + + /******************************************************************************* + * Setter for actionName + * @see #withActionName(String) + *******************************************************************************/ + public void setActionName(String actionName) + { + this.actionName = actionName; + } + + + + /******************************************************************************* + * Fluent setter for actionName + *******************************************************************************/ + public ProcessTraceBackendActivityStats withActionName(String actionName) + { + this.actionName = actionName; + return (this); + } + + + + /******************************************************************************* + * Getter for callCount + * @see #withCallCount(Integer) + *******************************************************************************/ + public Integer getCallCount() + { + return (this.callCount); + } + + + + /******************************************************************************* + * Setter for callCount + * @see #withCallCount(Integer) + *******************************************************************************/ + public void setCallCount(Integer callCount) + { + this.callCount = callCount; + } + + + + /******************************************************************************* + * Fluent setter for callCount + *******************************************************************************/ + public ProcessTraceBackendActivityStats withCallCount(Integer callCount) + { + this.callCount = callCount; + return (this); + } + + + + /******************************************************************************* + * Getter for recordCount + * @see #withRecordCount(Integer) + *******************************************************************************/ + public Integer getRecordCount() + { + return (this.recordCount); + } + + + + /******************************************************************************* + * Setter for recordCount + * @see #withRecordCount(Integer) + *******************************************************************************/ + public void setRecordCount(Integer recordCount) + { + this.recordCount = recordCount; + } + + + + /******************************************************************************* + * Fluent setter for recordCount + *******************************************************************************/ + public ProcessTraceBackendActivityStats withRecordCount(Integer recordCount) + { + this.recordCount = recordCount; + return (this); + } + + + + /******************************************************************************* + * Getter for runtimeMillis + * @see #withRuntimeMillis(Integer) + *******************************************************************************/ + public Integer getRuntimeMillis() + { + return (this.runtimeMillis); + } + + + + /******************************************************************************* + * Setter for runtimeMillis + * @see #withRuntimeMillis(Integer) + *******************************************************************************/ + public void setRuntimeMillis(Integer runtimeMillis) + { + this.runtimeMillis = runtimeMillis; + } + + + + /******************************************************************************* + * Fluent setter for runtimeMillis + *******************************************************************************/ + public ProcessTraceBackendActivityStats withRuntimeMillis(Integer runtimeMillis) + { + this.runtimeMillis = runtimeMillis; + return (this); + } + + + /******************************************************************************* + * Getter for qqqBackendId + * @see #withQqqBackendId(Integer) + *******************************************************************************/ + public Integer getQqqBackendId() + { + return (this.qqqBackendId); + } + + + + /******************************************************************************* + * Setter for qqqBackendId + * @see #withQqqBackendId(Integer) + *******************************************************************************/ + public void setQqqBackendId(Integer qqqBackendId) + { + this.qqqBackendId = qqqBackendId; + } + + + + /******************************************************************************* + * Fluent setter for qqqBackendId + *******************************************************************************/ + public ProcessTraceBackendActivityStats withQqqBackendId(Integer qqqBackendId) + { + this.qqqBackendId = qqqBackendId; + return (this); + } + + + + /******************************************************************************* + * Getter for qqqTableId + * @see #withQqqTableId(Integer) + *******************************************************************************/ + public Integer getQqqTableId() + { + return (this.qqqTableId); + } + + + + /******************************************************************************* + * Setter for qqqTableId + * @see #withQqqTableId(Integer) + *******************************************************************************/ + public void setQqqTableId(Integer qqqTableId) + { + this.qqqTableId = qqqTableId; + } + + + + /******************************************************************************* + * Fluent setter for qqqTableId + *******************************************************************************/ + public ProcessTraceBackendActivityStats withQqqTableId(Integer qqqTableId) + { + this.qqqTableId = qqqTableId; + return (this); + } + + +} diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTraceBackendActivityStatsAction.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTraceBackendActivityStatsAction.java new file mode 100644 index 0000000..f66f871 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/model/ProcessTraceBackendActivityStatsAction.java @@ -0,0 +1,132 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.model; + + +import java.util.Objects; +import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction; +import com.kingsrook.qqq.backend.core.actions.tables.GetAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction; +import com.kingsrook.qqq.backend.core.model.actions.audits.AuditDetailAccumulator; +import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.PossibleValueEnum; + + +/******************************************************************************* + ** ProcessTraceBackendActivityStatsAction - possible value enum + *******************************************************************************/ +public enum ProcessTraceBackendActivityStatsAction implements PossibleValueEnum +{ + INSERT(InsertAction.class.getSimpleName(), "Insert"), + UPDATE(UpdateAction.class.getSimpleName(), "Update"), + DELETE(DeleteAction.class.getSimpleName(), "Delete"), + QUERY(QueryAction.class.getSimpleName(), "Query"), + GET(GetAction.class.getSimpleName(), "Get"), + COUNT(CountAction.class.getSimpleName(), "Count"), + AGGREGATE(AuditDetailAccumulator.class.getSimpleName(), "Aggregate"); + + private final String id; + private final String label; + + public static final String NAME = "ProcessTraceBackendActivityStatsAction"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + ProcessTraceBackendActivityStatsAction(String id, String label) + { + this.id = id; + this.label = label; + } + + + + /******************************************************************************* + ** Get instance by id + ** + *******************************************************************************/ + public static ProcessTraceBackendActivityStatsAction getById(String id) + { + if(id == null) + { + return (null); + } + + for(ProcessTraceBackendActivityStatsAction value : ProcessTraceBackendActivityStatsAction.values()) + { + if(Objects.equals(value.id, id)) + { + return (value); + } + } + + return (null); + } + + + + /******************************************************************************* + ** Getter for id + ** + *******************************************************************************/ + public String getId() + { + return id; + } + + + + /******************************************************************************* + ** Getter for label + ** + *******************************************************************************/ + public String getLabel() + { + return label; + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueId() + { + return (getId()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Override + public String getPossibleValueLabel() + { + return (getLabel()); + } +} diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/ProcessTraceBackendActivityStatsManager.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/ProcessTraceBackendActivityStatsManager.java new file mode 100644 index 0000000..19bc5b5 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/ProcessTraceBackendActivityStatsManager.java @@ -0,0 +1,298 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.utils; + + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceBackendActivityStats; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.backends.QQQBackendTableManager; +import com.kingsrook.qqq.backend.core.model.tables.QQQTableTableManager; +import com.kingsrook.qqq.backend.core.utils.CollectionUtils; +import com.kingsrook.qqq.backend.core.utils.PrefixedDefaultThreadFactory; +import com.kingsrook.qqq.backend.core.utils.StringUtils; +import static com.kingsrook.qqq.backend.core.logging.LogUtils.logPair; + + +/******************************************************************************* + * Singleton that collects QueryStats, associated with a processTraceId. + * + *

Runs a scheduled executor thread to clean itself up - e.g., in case processes + * leak (e.g., started in a frontend but exited before final step). Configured by + * {@link #setCleanupIntervalMillis(long)} and {@link #setCleanupMaxAge(long)}

+ *******************************************************************************/ +public class ProcessTraceBackendActivityStatsManager +{ + private static final QLogger LOG = QLogger.getLogger(ProcessTraceBackendActivityStatsManager.class); + + private static ProcessTraceBackendActivityStatsManager processTraceBackendActivityStatsManager = null; + + private Map stats = Collections.synchronizedMap(new HashMap<>()); + + private long CLEANUP_INTERVAL_MILLIS = 60 * 60 * 1000; + private long CLEANUP_MAX_AGE = 12 * 60 * 60 * 1000; + + private ScheduledExecutorService scheduledExecutorService; + + + + /*************************************************************************** + * container for all data collected for a particular process trace. + * + * stores start milliseconds (for cleanup), plus map of keys to stats. + ***************************************************************************/ + private record ProcessStats(long startMillis, Map stats) + { + /******************************************************************************* + ** Constructor that uses default values + ** + *******************************************************************************/ + public ProcessStats() + { + this(System.currentTimeMillis(), new LinkedHashMap<>()); + } + } + + + + /*************************************************************************** + * what backend/table/action a set of stats are associated with. + * + * e.g., memoryBackend, personTable, insertAction + ***************************************************************************/ + public record Key(String backendName, String tableName, String actionName) implements Serializable + { + } + + + + /*************************************************************************** + * the stats collected for individual keys. + ***************************************************************************/ + private record Stats(Integer calls, Integer records, Integer millis) + { + } + + + + /******************************************************************************* + ** Singleton constructor + *******************************************************************************/ + private ProcessTraceBackendActivityStatsManager() + { + scheduleCleanupJob(); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private void scheduleCleanupJob() + { + scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new PrefixedDefaultThreadFactory("ProcessTraceBackendActivityStatsManager-cleanup")); + scheduledExecutorService.scheduleAtFixedRate(this::cleanup, CLEANUP_INTERVAL_MILLIS, CLEANUP_INTERVAL_MILLIS, TimeUnit.MILLISECONDS); + } + + + + /******************************************************************************* + ** Singleton accessor + *******************************************************************************/ + public static ProcessTraceBackendActivityStatsManager getInstance() + { + if(processTraceBackendActivityStatsManager == null) + { + processTraceBackendActivityStatsManager = new ProcessTraceBackendActivityStatsManager(); + } + return (processTraceBackendActivityStatsManager); + } + + + + /*************************************************************************** + * shutdown the scheduled job. also clear stats. + ***************************************************************************/ + public void shutdown() + { + scheduledExecutorService.shutdown(); + stats.clear(); + } + + + + /*************************************************************************** + * initialize storage for stats for a specific process trace id + ***************************************************************************/ + public void initProcess(Long processTraceId) + { + stats.put(processTraceId, new ProcessStats()); + } + + + + /*************************************************************************** + * clean up stat entries for processes that were originally created before + * CLEANUP_MAX_AGE (per {@link #setCleanupMaxAge(long)} milliseconds ago. + ***************************************************************************/ + private void cleanup() + { + long limitStartMillis = System.currentTimeMillis() - CLEANUP_MAX_AGE; + LOG.info("Starting cleanup of ProcessTraceBackendActivityStatsManager", logPair("size", stats.size())); + stats.entrySet().removeIf(entry -> entry.getValue().startMillis < limitStartMillis); + LOG.info("Finished cleanup of ProcessTraceBackendActivityStatsManager", logPair("size", stats.size())); + } + + + + /*************************************************************************** + * when storing a processTrace (e.g., after the process has finished running), + * convert the stats stored in this object to a list of {@link ProcessTraceBackendActivityStats} + * entities. Also remove the stats from this object's internal storage. + ***************************************************************************/ + public List getAndRemoveStats(Long processTraceId) throws QException + { + List rs = new ArrayList<>(); + + ProcessStats processStats = stats.remove(processTraceId); + if(processStats != null) + { + for(Map.Entry entry : CollectionUtils.nonNullMap(processStats.stats()).entrySet()) + { + rs.add(new ProcessTraceBackendActivityStats() + .withProcessTraceId(processTraceId) + .withQqqBackendId(getBackendId(entry.getKey().backendName())) + .withQqqTableId(getTableId(entry.getKey().tableName())) + .withActionName(entry.getKey().actionName()) + .withCallCount(entry.getValue().calls()) + .withRecordCount(entry.getValue().records()) + .withRuntimeMillis(entry.getValue().millis()) + ); + } + } + + return (rs); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private Integer getBackendId(String backendName) + { + if(!StringUtils.hasContent(backendName)) + { + return (null); + } + + try + { + return QQQBackendTableManager.getQQQBackendId(QContext.getQInstance(), backendName); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private Integer getTableId(String tableName) + { + if(!StringUtils.hasContent(tableName)) + { + return (null); + } + + try + { + return QQQTableTableManager.getQQQTableId(QContext.getQInstance(), tableName); + } + catch(Exception e) + { + return (null); + } + } + + + + /*************************************************************************** + * increment the stat values for a given process. + * + * @param key the backend, table, and action + * @param addCalls the number of calls to add (typically 1 at a time?) + * @param addRecords the number of records to add + * @param addMillis the number of milliseconds of runtime to add + ***************************************************************************/ + public void add(Long processTraceId, Key key, Integer addCalls, Integer addRecords, Integer addMillis) + { + ProcessStats processStats = stats.computeIfAbsent(processTraceId, k -> new ProcessStats()); + Map countsForProcess = processStats.stats(); + + if(countsForProcess != null) + { + Stats statsForKey = countsForProcess.computeIfAbsent(key, (k) -> new Stats(0, 0, 0)); + countsForProcess.put(key, new Stats(statsForKey.calls + addCalls, statsForKey.records + addRecords, statsForKey.millis + addMillis)); + } + } + + + + /*************************************************************************** + * sets the cleanup job's scheduled interval. Also reschedules the job. + * + * @param cleanupIntervalMillis milliseconds between runs of the cleanup job + ***************************************************************************/ + public void setCleanupIntervalMillis(long cleanupIntervalMillis) + { + CLEANUP_INTERVAL_MILLIS = cleanupIntervalMillis; + scheduledExecutorService.shutdown(); + scheduleCleanupJob(); + } + + + + /*************************************************************************** + * @param cleanupMaxAge max age (in millis) that a process is allowed to be + * before it gets cleaned up when a cleanup job runs. + ***************************************************************************/ + public void setCleanupMaxAge(long cleanupMaxAge) + { + CLEANUP_MAX_AGE = cleanupMaxAge; + } + +} diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTraceQueryStatConsumer.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTraceQueryStatConsumer.java new file mode 100644 index 0000000..e60acf2 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTraceQueryStatConsumer.java @@ -0,0 +1,67 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.utils; + + +import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatConsumerInterface; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.model.querystats.QueryStat; + + +/******************************************************************************* + * Standard Process Trace's implementation of a QueryStat consumer. + * + *

That is, to accumulate them in the {@link ProcessTraceBackendActivityStatsManager}, + * associated with the processTraceId in the user's session, so they can be stored + * when the process trace is stored.

+ *******************************************************************************/ +public class StandardProcessTraceQueryStatConsumer implements QueryStatConsumerInterface +{ + private static final QLogger LOG = QLogger.getLogger(StandardProcessTraceQueryStatConsumer.class); + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void accept(QueryStat queryStat) + { + try + { + String processTraceId = QContext.getQSession().getValue(StandardProcessTracer.PROCESS_TRACE_ID_SESSION_KEY); + if(processTraceId == null) + { + return; + } + + String backendName = QContext.getQInstance().getBackendForTable(queryStat.getTableName()).getName(); + String backendAction = queryStat.getBackendAction(); + + ProcessTraceBackendActivityStatsManager.Key key = new ProcessTraceBackendActivityStatsManager.Key(backendName, queryStat.getTableName(), backendAction); + ProcessTraceBackendActivityStatsManager.getInstance().add(Long.valueOf(processTraceId), key, 1, queryStat.getRecordCount(), queryStat.getFirstResultMillis()); + } + catch(Exception e) + { + LOG.warn("Error accepting query stats for process trace", e); + } + } +} diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTracer.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTracer.java index f43096c..dd6bcf3 100644 --- a/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTracer.java +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTracer.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import com.kingsrook.qbits.standardprocesstrace.StandardProcessTraceQBitConfig; import com.kingsrook.qbits.standardprocesstrace.model.ProcessTrace; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceBackendActivityStats; import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceSummaryLine; import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceSummaryLineRecordInt; import com.kingsrook.qqq.backend.core.actions.tables.GetAction; @@ -75,6 +76,8 @@ public class StandardProcessTracer implements ProcessTracerInterface private static StandardProcessTraceQBitConfig standardProcessTraceQBitConfig; + public static final String PROCESS_TRACE_ID_SESSION_KEY = StandardProcessTracer.class.getName() + ".processTraceId"; + private Instant startTime; private Long processTraceId; @@ -98,6 +101,17 @@ public class StandardProcessTracer implements ProcessTracerInterface .withProcessUUID(runProcessInput.getProcessUUID()))); processTraceId = insertOutput.getRecords().get(0).getValueLong("id"); + + //////////////////////////////////////////////////////////////////////////////////// + // store the processTraceId in the session. this will be used by the // + // ProcessTraceQueryStatConsumer to associated QueryStats with this process trace // + //////////////////////////////////////////////////////////////////////////////////// + if(QContext.getQSession() != null) + { + QContext.getQSession().setValue(PROCESS_TRACE_ID_SESSION_KEY, String.valueOf(processTraceId)); + } + + ProcessTraceBackendActivityStatsManager.getInstance().initProcess(processTraceId); } catch(QException e) { @@ -113,9 +127,29 @@ public class StandardProcessTracer implements ProcessTracerInterface @Override public void handleProcessResume(RunProcessInput runProcessInput) { - ///////////////////////////////// - // noop in this implementation // - ///////////////////////////////// + try + { + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // if a user is resuming a process, e.g., from a frontend, then we want to put that processTraceId in their session // + ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + if(processTraceId == null) + { + QRecord processTrace = GetAction.execute(ProcessTrace.TABLE_NAME, Map.of("processUUID", runProcessInput.getProcessUUID())); + if(processTrace != null) + { + processTraceId = processTrace.getValueLong("id"); + } + } + + if(processTraceId != null && QContext.getQSession() != null) + { + QContext.getQSession().setValue(PROCESS_TRACE_ID_SESSION_KEY, String.valueOf(processTraceId)); + } + } + catch(Exception e) + { + LOG.warn("Error resuming process trace", e, logPair("processUUID", () -> runProcessInput.getProcessUUID())); + } } @@ -328,12 +362,28 @@ public class StandardProcessTracer implements ProcessTracerInterface new InsertAction().execute(new InsertInput(ProcessTraceSummaryLine.TABLE_NAME).withRecordEntities(summaryLines)); } + + ////////////////////////////////////////// + // build and store backend stat records // + ////////////////////////////////////////// + if(QContext.getQInstance().getTable(ProcessTraceBackendActivityStats.TABLE_NAME) != null) + { + List stats = ProcessTraceBackendActivityStatsManager.getInstance().getAndRemoveStats(processTraceId); + new InsertAction().execute(new InsertInput(ProcessTraceBackendActivityStats.TABLE_NAME).withRecordEntities(stats)); + } } } catch(Exception e) { LOG.warn("Error completing store of processTrace records.", e); } + finally + { + if(QContext.getQSession() != null) + { + QContext.getQSession().removeValue(PROCESS_TRACE_ID_SESSION_KEY); + } + } } -- 2.54.0 From 6d283a2ca49a85c6a6a3530b598c60592718d624 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 4 Nov 2025 16:55:44 -0600 Subject: [PATCH 2/5] refactor: update this qbit config and producer to follow current interfaces from qqq also add config for includeBackendActivityStats --- .../StandardProcessTraceQBitConfig.java | 93 ++++++++++++++- .../StandardProcessTraceQBitProducer.java | 110 +++++++++--------- 2 files changed, 147 insertions(+), 56 deletions(-) diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceQBitConfig.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceQBitConfig.java index 2026411..d5bc8b6 100644 --- a/src/main/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceQBitConfig.java +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceQBitConfig.java @@ -22,16 +22,25 @@ package com.kingsrook.qbits.standardprocesstrace; +import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitConfig; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + /******************************************************************************* - ** ad-hoc Configuration data for this qbit. - ** TODO - when qbit configs become formalized in qqq, adapt this qbit to that interface. + * Configuration data for this qbit. + * *******************************************************************************/ -public class StandardProcessTraceQBitConfig +public class StandardProcessTraceQBitConfig implements QBitConfig { + private MetaDataCustomizerInterface tableMetaDataCustomizer; + private String userIdPossibleValueSourceName; private String userTableName; private String userIdReferenceFieldName; + private boolean includeBackendActivityStats = false; + /******************************************************************************* @@ -125,4 +134,82 @@ public class StandardProcessTraceQBitConfig } + + /******************************************************************************* + * Getter for includeBackendActivityStats + * @see #withIncludeBackendActivityStats(boolean) + *******************************************************************************/ + public boolean getIncludeBackendActivityStats() + { + return (this.includeBackendActivityStats); + } + + + + /******************************************************************************* + * Setter for includeBackendActivityStats + * @see #withIncludeBackendActivityStats(boolean) + *******************************************************************************/ + public void setIncludeBackendActivityStats(boolean includeBackendActivityStats) + { + this.includeBackendActivityStats = includeBackendActivityStats; + } + + + + /******************************************************************************* + * Fluent setter for includeBackendActivityStats + * + * @param includeBackendActivityStats + * indicate whether or not the ProcessTraceBackendActivityStats table should be + * included in the QInstance (and records should be built in that table under + * processTrace records). + *

Note that using this feature requires the QQQ QueryStatManager to be active + * in the application server.

+ * @return this + *******************************************************************************/ + public StandardProcessTraceQBitConfig withIncludeBackendActivityStats(boolean includeBackendActivityStats) + { + this.includeBackendActivityStats = includeBackendActivityStats; + return (this); + } + + + + /******************************************************************************* + * Getter for tableMetaDataCustomizer + * @see #withTableMetaDataCustomizer(MetaDataCustomizerInterface) + *******************************************************************************/ + public MetaDataCustomizerInterface getTableMetaDataCustomizer() + { + return (this.tableMetaDataCustomizer); + } + + + + /******************************************************************************* + * Setter for tableMetaDataCustomizer + * @see #withTableMetaDataCustomizer(MetaDataCustomizerInterface) + *******************************************************************************/ + public void setTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + this.tableMetaDataCustomizer = tableMetaDataCustomizer; + } + + + + /******************************************************************************* + * Fluent setter for tableMetaDataCustomizer + * + * @param tableMetaDataCustomizer + * table meta data customizer to be applied to all tables produced by this qbit. + * @return this + *******************************************************************************/ + public StandardProcessTraceQBitConfig withTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) + { + this.tableMetaDataCustomizer = tableMetaDataCustomizer; + return (this); + } + + } diff --git a/src/main/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceQBitProducer.java b/src/main/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceQBitProducer.java index 911b5e4..e5b68a9 100644 --- a/src/main/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceQBitProducer.java @@ -23,15 +23,15 @@ package com.kingsrook.qbits.standardprocesstrace; import com.kingsrook.qbits.standardprocesstrace.model.ProcessTrace; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceBackendActivityStats; import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceSummaryLine; import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceSummaryLineRecordInt; import com.kingsrook.qqq.backend.core.exceptions.QException; -import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerHelper; -import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerInterface; import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; -import com.kingsrook.qqq.backend.core.model.metadata.producers.MetaDataCustomizerInterface; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitMetaDataProducer; import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; import com.kingsrook.qqq.backend.core.utils.StringUtils; @@ -39,10 +39,9 @@ import com.kingsrook.qqq.backend.core.utils.StringUtils; /******************************************************************************* ** *******************************************************************************/ -public class StandardProcessTraceQBitProducer implements MetaDataProducerInterface +public class StandardProcessTraceQBitProducer implements QBitMetaDataProducer { - private StandardProcessTraceQBitConfig standardProcessTraceQBitConfig; - private MetaDataCustomizerInterface tableMetaDataCustomizer; + private StandardProcessTraceQBitConfig standardProcessTraceQBitConfig; @@ -51,35 +50,71 @@ public class StandardProcessTraceQBitProducer implements MetaDataProducerInterfa ***************************************************************************/ public static QAppSection getAppSection(QInstance qInstance) { - return (new QAppSection().withName("processTrace") + QBitMetaData qBitMetaData = qInstance.getQBits().get(new StandardProcessTraceQBitProducer().getQBitMetaData().getName()); + + QAppSection section = new QAppSection().withName("processTrace") .withTable(ProcessTrace.TABLE_NAME) .withTable(ProcessTraceSummaryLine.TABLE_NAME) - .withTable(ProcessTraceSummaryLineRecordInt.TABLE_NAME)); + .withTable(ProcessTraceSummaryLineRecordInt.TABLE_NAME); + + if(qBitMetaData != null && qBitMetaData.getConfig() instanceof StandardProcessTraceQBitConfig config) + { + if(config.getIncludeBackendActivityStats()) + { + section.withTable(ProcessTraceBackendActivityStats.TABLE_NAME); + } + } + + return (section); } /*************************************************************************** - ** + * ***************************************************************************/ @Override - public MetaDataProducerMultiOutput produce(QInstance qInstance) throws QException + public StandardProcessTraceQBitConfig getQBitConfig() { - MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, ProcessTrace.class.getPackageName(), tableMetaDataCustomizer); - MetaDataProducerHelper.processAllMetaDataProducersInPackage(qInstance, getClass().getPackageName() + ".metadata", tableMetaDataCustomizer); + return (standardProcessTraceQBitConfig); + } - ////////////////////////////////////////////////////////////////////////////////////// - // set the possible value source on the processTrace.userId field, if so configured // - ////////////////////////////////////////////////////////////////////////////////////// - if(standardProcessTraceQBitConfig != null && StringUtils.hasContent(standardProcessTraceQBitConfig.getUserIdPossibleValueSourceName())) + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public QBitMetaData getQBitMetaData() + { + QBitMetaData qBitMetaData = new QBitMetaData() + .withGroupId("com.kingsrook.qbits") + .withArtifactId("standard-process-trace") + .withVersion("0.1.3") + .withNamespace(getNamespace()) + .withConfig(getQBitConfig()); + + return qBitMetaData; + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void postProduceActions(MetaDataProducerMultiOutput metaDataProducerMultiOutput, QInstance qInstance) throws QException + { + if(standardProcessTraceQBitConfig != null) { - qInstance.getTable(ProcessTrace.TABLE_NAME).getField("userId").setPossibleValueSourceName(standardProcessTraceQBitConfig.getUserIdPossibleValueSourceName()); + ////////////////////////////////////////////////////////////////////////////////////// + // set the possible value source on the processTrace.userId field, if so configured // + ////////////////////////////////////////////////////////////////////////////////////// + if(StringUtils.hasContent(standardProcessTraceQBitConfig.getUserIdPossibleValueSourceName())) + { + metaDataProducerMultiOutput.get(QTableMetaData.class, ProcessTrace.TABLE_NAME).getField("userId").setPossibleValueSourceName(standardProcessTraceQBitConfig.getUserIdPossibleValueSourceName()); + } } - - //////////////////////////////////////////////////////////////////////////////////////////////////// - // we actually called MetaDataProducerHelper ourselves, so, don't have anything to return here... // - //////////////////////////////////////////////////////////////////////////////////////////////////// - return new MetaDataProducerMultiOutput(); } @@ -113,35 +148,4 @@ public class StandardProcessTraceQBitProducer implements MetaDataProducerInterfa return (this); } - - - /******************************************************************************* - ** Getter for tableMetaDataCustomizer - *******************************************************************************/ - public MetaDataCustomizerInterface getTableMetaDataCustomizer() - { - return (this.tableMetaDataCustomizer); - } - - - - /******************************************************************************* - ** Setter for tableMetaDataCustomizer - *******************************************************************************/ - public void setTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) - { - this.tableMetaDataCustomizer = tableMetaDataCustomizer; - } - - - - /******************************************************************************* - ** Fluent setter for tableMetaDataCustomizer - *******************************************************************************/ - public StandardProcessTraceQBitProducer withTableMetaDataCustomizer(MetaDataCustomizerInterface tableMetaDataCustomizer) - { - this.tableMetaDataCustomizer = tableMetaDataCustomizer; - return (this); - } - } -- 2.54.0 From 5a44c5d1a8550ad56ed001f03b9b9440b797b84b Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 4 Nov 2025 16:56:08 -0600 Subject: [PATCH 3/5] test: Add some tests (ya, this qbit had none...) --- .../qbits/standardprocesstrace/BaseTest.java | 86 +++++++ .../StandardProcessTraceTestApplication.java | 234 ++++++++++++++++++ ...sTraceBackendActivityStatsManagerTest.java | 115 +++++++++ .../utils/StandardProcessTracerTest.java | 177 +++++++++++++ 4 files changed, 612 insertions(+) create mode 100644 src/test/java/com/kingsrook/qbits/standardprocesstrace/BaseTest.java create mode 100644 src/test/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceTestApplication.java create mode 100644 src/test/java/com/kingsrook/qbits/standardprocesstrace/utils/ProcessTraceBackendActivityStatsManagerTest.java create mode 100644 src/test/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTracerTest.java diff --git a/src/test/java/com/kingsrook/qbits/standardprocesstrace/BaseTest.java b/src/test/java/com/kingsrook/qbits/standardprocesstrace/BaseTest.java new file mode 100644 index 0000000..e033216 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/standardprocesstrace/BaseTest.java @@ -0,0 +1,86 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace; + + +import java.util.List; +import com.kingsrook.qbits.standardprocesstrace.utils.StandardProcessTraceQueryStatConsumer; +import com.kingsrook.qqq.backend.core.actions.tables.helpers.QueryStatManager; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.QInstanceValidator; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.session.QSystemUserSession; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; + + +/******************************************************************************* + * base class for tests for this QBit. + *******************************************************************************/ +public class BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @BeforeEach + void baseBeforeEach() throws Exception + { + QInstance qInstance = defineQInstance(); + new QInstanceValidator().validate(qInstance); + QContext.init(qInstance, new QSystemUserSession()); + + QueryStatManager queryStatManager = QueryStatManager.getInstance(); + queryStatManager.start(qInstance, () -> new QSystemUserSession()); + queryStatManager.setQueryStatConsumers(List.of(new StandardProcessTraceQueryStatConsumer())); + + MemoryRecordStore.fullReset(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + MemoryRecordStore.fullReset(); + + QueryStatManager.getInstance().stop(); + } + + + + /*************************************************************************** + ** + ***************************************************************************/ + private QInstance defineQInstance() throws QException + { + return new StandardProcessTraceTestApplication().defineQInstance(); + } + + + +} diff --git a/src/test/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceTestApplication.java b/src/test/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceTestApplication.java new file mode 100644 index 0000000..70ac661 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/standardprocesstrace/StandardProcessTraceTestApplication.java @@ -0,0 +1,234 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace; + + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import com.kingsrook.qbits.standardprocesstrace.utils.StandardProcessTracer; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.instances.AbstractQQQApplication; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine; +import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput; +import com.kingsrook.qqq.backend.core.model.actions.processes.Status; +import com.kingsrook.qqq.backend.core.model.backends.QQQBackendsMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput; +import com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType; +import com.kingsrook.qqq.backend.core.model.metadata.QBackendMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.audits.AuditLevel; +import com.kingsrook.qqq.backend.core.model.metadata.audits.QAuditRules; +import com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppSection; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon; +import com.kingsrook.qqq.backend.core.model.metadata.tables.Capability; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import com.kingsrook.qqq.backend.core.model.processes.QQQProcessesMetaDataProvider; +import com.kingsrook.qqq.backend.core.model.tables.QQQTablesMetaDataProvider; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryBackendModule; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.LoadViaInsertStep; +import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess; + + +/******************************************************************************* + * test application (e.g., QInstance) for this QBit's tests. + *******************************************************************************/ +public class StandardProcessTraceTestApplication extends AbstractQQQApplication +{ + public static final String MEMORY_BACKEND_NAME = "memory"; + + public static final String TABLE_NAME_PERSON = "person"; + public static final String TABLE_NAME_ORDER = "order"; + + public static final String PROCESS_NAME_PLACE_ORDERS = "placeOrders"; + + + + /*************************************************************************** + ** + ***************************************************************************/ + @Override + public QInstance defineQInstance() throws QException + { + QInstance qInstance = new QInstance(); + + qInstance.setAuthentication(new QAuthenticationMetaData().withType(QAuthenticationType.FULLY_ANONYMOUS)); + + qInstance.addBackend(new QBackendMetaData() + .withName(MEMORY_BACKEND_NAME) + .withBackendType(MemoryBackendModule.class)); + + ////////////////////////////// + // add tables table and PVS // + ////////////////////////////// + new QQQTablesMetaDataProvider().defineAll(qInstance, MEMORY_BACKEND_NAME, MEMORY_BACKEND_NAME, null); + new QQQProcessesMetaDataProvider().defineAll(qInstance, MEMORY_BACKEND_NAME, MEMORY_BACKEND_NAME, null); + new QQQBackendsMetaDataProvider().defineAll(qInstance, MEMORY_BACKEND_NAME, MEMORY_BACKEND_NAME, null); + + ////////////////////// + // produce our qbit // + ////////////////////// + StandardProcessTraceQBitProducer producer = new StandardProcessTraceQBitProducer() + .withStandardProcessTraceQBitConfig(new StandardProcessTraceQBitConfig() + .withTableMetaDataCustomizer((QInstance instance, QTableMetaData table) -> + { + table.setBackendName(MEMORY_BACKEND_NAME); + return (table); + }) + .withIncludeBackendActivityStats(true)); + + MetaDataProducerMultiOutput allQBitMetaData = producer.produce(qInstance); + allQBitMetaData.addSelfToInstance(qInstance); + + //////////////////////////////////////// + // produce some test tables // + // note - nice to be a qbit this too? // + //////////////////////////////////////// + qInstance.addTable(new QTableMetaData() + .withName(TABLE_NAME_PERSON) + .withBackendName(MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("firstName", QFieldType.STRING).withIsEditable(true)) + .withField(new QFieldMetaData("lastName", QFieldType.STRING).withIsEditable(true)) + .withCapability(Capability.QUERY_STATS) + ); + + qInstance.addTable(new QTableMetaData() + .withName(TABLE_NAME_ORDER) + .withBackendName(MEMORY_BACKEND_NAME) + .withPrimaryKeyField("id") + .withField(new QFieldMetaData("id", QFieldType.INTEGER).withIsEditable(false)) + .withField(new QFieldMetaData("orderNo", QFieldType.STRING).withIsEditable(true)) + .withField(new QFieldMetaData("storeId", QFieldType.INTEGER).withIsEditable(true)) + .withField(new QFieldMetaData("shipTo", QFieldType.STRING).withIsEditable(true)) + .withCapability(Capability.QUERY_STATS) + ); + + qInstance.addProcess(StreamedETLWithFrontendProcess.processMetaDataBuilder() + .withName(PROCESS_NAME_PLACE_ORDERS) + .withTableName(TABLE_NAME_PERSON) + .withSourceTable(TABLE_NAME_PERSON) + .withDestinationTable(TABLE_NAME_ORDER) + .withExtractStepClass(ExtractViaQueryStep.class) + .withTransformStepClass(PlaceOrdersProcessTransformStep.class) + .withLoadStepClass(LoadViaInsertStep.class) + .getProcessMetaData() + .withProcessTracerCodeReference(new QCodeReference(StandardProcessTracer.class)) + ); + + /////////////////////////////////////////// + // turn off audits (why on by default??) // + /////////////////////////////////////////// + qInstance.getTables().values().forEach(t -> t.setAuditRules(new QAuditRules().withAuditLevel(AuditLevel.NONE))); + + ///////////////// + // create apps // + ///////////////// + qInstance.addApp(new QAppMetaData() + .withName("logs") + .withIcon(new QIcon("history_edu")) + .withSection(StandardProcessTraceQBitProducer.getAppSection(qInstance))); + + qInstance.addApp(new QAppMetaData() + .withName("testData") + .withIcon(new QIcon("dataset")) + .withSection(new QAppSection() + .withName("people") + .withTables(List.of(TABLE_NAME_PERSON)))); + + return qInstance; + } + + + + /*************************************************************************** + * test process transformer - for each input person, creates an output order (to insert) + ***************************************************************************/ + public static class PlaceOrdersProcessTransformStep extends AbstractTransformStep + { + private ProcessSummaryLine okLine = new ProcessSummaryLine(Status.OK) + .withMessageSuffix("") + .withSingularFutureMessage("will get a new order") + .withPluralFutureMessage("will get new orders") + .withSingularPastMessage("got a new order") + .withPluralPastMessage("got new orders"); + + private ProcessSummaryLine errorLine = new ProcessSummaryLine(Status.ERROR) + .withMessageSuffix("") + .withSingularFutureMessage("will not get a new order") + .withPluralFutureMessage("will not get new orders") + .withSingularPastMessage("did not get a new order") + .withPluralPastMessage("did not get new orders"); + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public ArrayList getProcessSummary(RunBackendStepOutput runBackendStepOutput, boolean isForResultScreen) + { + ArrayList rs = new ArrayList<>(); + okLine.addSelfToListIfAnyCount(rs); + errorLine.addSelfToListIfAnyCount(rs); + return rs; + } + + + + /*************************************************************************** + * + ***************************************************************************/ + @Override + public void runOnePage(RunBackendStepInput runBackendStepInput, RunBackendStepOutput runBackendStepOutput) throws QException + { + for(QRecord personRecord : runBackendStepInput.getRecords()) + { + //////////////////////////////////////// + // don't let Mr. Burns order anything // + //////////////////////////////////////// + if("Burns".equals(personRecord.getValueString("lastName"))) + { + errorLine.incrementCountAndAddPrimaryKey(personRecord.getValue("id")); + continue; + } + + QRecord orderRecord = new QRecord() + .withValue("orderNo", UUID.randomUUID().toString()) + .withValue("shipTo", personRecord.getValueString("firstName") + " " + personRecord.getValueString("lastName")); + runBackendStepOutput.addRecord(orderRecord); + okLine.incrementCountAndAddPrimaryKey(personRecord.getValue("id")); + } + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/standardprocesstrace/utils/ProcessTraceBackendActivityStatsManagerTest.java b/src/test/java/com/kingsrook/qbits/standardprocesstrace/utils/ProcessTraceBackendActivityStatsManagerTest.java new file mode 100644 index 0000000..415c105 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/standardprocesstrace/utils/ProcessTraceBackendActivityStatsManagerTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.utils; + + +import java.util.List; +import java.util.concurrent.TimeUnit; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceBackendActivityStats; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.logging.QCollectingLogger; +import com.kingsrook.qqq.backend.core.logging.QLogger; +import com.kingsrook.qqq.backend.core.utils.SleepUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + + +/******************************************************************************* + ** Unit test for ProcessTraceBackendActivityStatsManager + *******************************************************************************/ +class ProcessTraceBackendActivityStatsManagerTest +{ + ProcessTraceBackendActivityStatsManager.Key key = new ProcessTraceBackendActivityStatsManager.Key("b", "t", "a"); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void afterEach() + { + ProcessTraceBackendActivityStatsManager manager = ProcessTraceBackendActivityStatsManager.getInstance(); + manager.shutdown(); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMaxAgeLarger() throws QException + { + long processTraceId = 1L; + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(ProcessTraceBackendActivityStatsManager.class); + + ProcessTraceBackendActivityStatsManager manager = ProcessTraceBackendActivityStatsManager.getInstance(); + manager.initProcess(processTraceId); + manager.add(processTraceId, key, 1, 2, 3); + manager.setCleanupIntervalMillis(10); + manager.setCleanupMaxAge(100); + SleepUtils.sleep(50, TimeUnit.MILLISECONDS); + + QLogger.deactivateCollectingLoggerForClass(ProcessTraceBackendActivityStatsManager.class); + assertThat(collectingLogger.getCollectedMessages()).allMatch(clm -> clm.getMessage().contains("\"size\":1")); + + List stats = manager.getAndRemoveStats(processTraceId); + assertEquals(1, stats.size()); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testMaxAgeSmaller() throws QException + { + long processTraceId = 2L; + QCollectingLogger collectingLogger = QLogger.activateCollectingLoggerForClass(ProcessTraceBackendActivityStatsManager.class); + + ProcessTraceBackendActivityStatsManager manager = ProcessTraceBackendActivityStatsManager.getInstance(); + manager.initProcess(processTraceId); + manager.add(processTraceId, key, 1, 2, 3); + manager.setCleanupIntervalMillis(5); + manager.setCleanupMaxAge(20); + SleepUtils.sleep(50, TimeUnit.MILLISECONDS); + + QLogger.deactivateCollectingLoggerForClass(ProcessTraceBackendActivityStatsManager.class); + assertThat(collectingLogger.getCollectedMessages()).anyMatch(clm -> clm.getMessage().contains("\"size\":1")); + assertThat(collectingLogger.getCollectedMessages()).anyMatch(clm -> clm.getMessage().contains("\"size\":0")); + + List stats = manager.getAndRemoveStats(processTraceId); + assertEquals(0, stats.size()); + + //////////////////////////////////////////////////////////////////////////////////////////// + // make sure even after a cleanup, we don't fail if we do try to add values for a process // + //////////////////////////////////////////////////////////////////////////////////////////// + manager.add(processTraceId, key, 1, 2, 3); + stats = manager.getAndRemoveStats(processTraceId); + assertEquals(1, stats.size()); + } + +} \ No newline at end of file diff --git a/src/test/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTracerTest.java b/src/test/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTracerTest.java new file mode 100644 index 0000000..3944203 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/standardprocesstrace/utils/StandardProcessTracerTest.java @@ -0,0 +1,177 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.kingsrook.qbits.standardprocesstrace.utils; + + +import java.util.List; +import com.kingsrook.qbits.standardprocesstrace.BaseTest; +import com.kingsrook.qbits.standardprocesstrace.StandardProcessTraceTestApplication; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTrace; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceBackendActivityStats; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceSummaryLine; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceSummaryLineRecordInt; +import com.kingsrook.qbits.standardprocesstrace.model.ProcessTraceSummaryStatusEnum; +import com.kingsrook.qqq.backend.core.actions.processes.QProcessCallbackFactory; +import com.kingsrook.qqq.backend.core.actions.processes.RunProcessAction; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessInput; +import com.kingsrook.qqq.backend.core.model.actions.processes.RunProcessOutput; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.model.tables.QQQTableTableManager; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + + +/******************************************************************************* + ** Unit test for StandardProcessTracer + *******************************************************************************/ +class StandardProcessTracerTest extends BaseTest +{ + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void test() throws QException + { + insertHomerMargeAndMrBurns(); + + //////////////////////////////////////// + // run process skipping backend steps // + //////////////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(StandardProcessTraceTestApplication.PROCESS_NAME_PLACE_ORDERS); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.SKIP); + new RunProcessAction().execute(input); + + baseAssertionsAfterProcess(); + + Integer personTableId = QQQTableTableManager.getQQQTableId(QContext.getQInstance(), StandardProcessTraceTestApplication.TABLE_NAME_PERSON); + Integer orderTableId = QQQTableTableManager.getQQQTableId(QContext.getQInstance(), StandardProcessTraceTestApplication.TABLE_NAME_ORDER); + + List processTraceBackendActivityStatRecords = QueryAction.execute(ProcessTraceBackendActivityStats.TABLE_NAME, new QQueryFilter()); + assertThat(processTraceBackendActivityStatRecords) + .hasSizeGreaterThanOrEqualTo(2) + .anyMatch(r -> r.getValue("qqqTableId").equals(personTableId) && r.getValue("actionName").equals(QueryAction.class.getSimpleName()) && r.getValueInteger("callCount").equals(1) && r.getValueInteger("recordCount").equals(3)) + .anyMatch(r -> r.getValue("qqqTableId").equals(orderTableId) && r.getValue("actionName").equals(InsertAction.class.getSimpleName()) && r.getValueInteger("callCount").equals(1) && r.getValueInteger("recordCount").equals(2)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void testResumingFrontendProcess() throws QException + { + insertHomerMargeAndMrBurns(); + + ////////////////////////////////////// + // start process - break at preview // + ////////////////////////////////////// + RunProcessInput input = new RunProcessInput(); + input.setProcessName(StandardProcessTraceTestApplication.PROCESS_NAME_PLACE_ORDERS); + input.setCallback(QProcessCallbackFactory.forFilter(new QQueryFilter())); + input.setFrontendStepBehavior(RunProcessInput.FrontendStepBehavior.BREAK); + RunProcessOutput output = new RunProcessAction().execute(input); + + ///////////////////////////////////// + // continue to execute the process // + ///////////////////////////////////// + input.setProcessUUID(output.getProcessUUID()); + input.setStartAfterStep(output.getProcessState().getNextStepName().get()); + new RunProcessAction().execute(input); + + baseAssertionsAfterProcess(); + + Integer personTableId = QQQTableTableManager.getQQQTableId(QContext.getQInstance(), StandardProcessTraceTestApplication.TABLE_NAME_PERSON); + Integer orderTableId = QQQTableTableManager.getQQQTableId(QContext.getQInstance(), StandardProcessTraceTestApplication.TABLE_NAME_ORDER); + + List processTraceBackendActivityStatRecords = QueryAction.execute(ProcessTraceBackendActivityStats.TABLE_NAME, new QQueryFilter()); + assertThat(processTraceBackendActivityStatRecords) + .hasSizeGreaterThanOrEqualTo(2) + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + // this time, there'll be 2 calls to the query (for preview, and execute steps), for twice as many records total // + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + .anyMatch(r -> r.getValue("qqqTableId").equals(personTableId) && r.getValue("actionName").equals(QueryAction.class.getSimpleName()) && r.getValueInteger("callCount").equals(2) && r.getValueInteger("recordCount").equals(6)) + .anyMatch(r -> r.getValue("qqqTableId").equals(orderTableId) && r.getValue("actionName").equals(InsertAction.class.getSimpleName()) && r.getValueInteger("callCount").equals(1) && r.getValueInteger("recordCount").equals(2)); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private static void insertHomerMargeAndMrBurns() throws QException + { + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + // insert 3 people to run through the process. // + // the process will insert an order for Homer & Marge, but is hard-coded to make an error for Mr. Burns. // + /////////////////////////////////////////////////////////////////////////////////////////////////////////// + new InsertAction().execute(new InsertInput(StandardProcessTraceTestApplication.TABLE_NAME_PERSON).withRecords(List.of( + new QRecord().withValue("id", 100).withValue("firstName", "Homer").withValue("lastName", "Simpson"), + new QRecord().withValue("id", 101).withValue("firstName", "Marge").withValue("lastName", "Simpson"), + new QRecord().withValue("id", 102).withValue("firstName", "Mr.").withValue("lastName", "Burns") + ))); + } + + + + /*************************************************************************** + * + ***************************************************************************/ + private static void baseAssertionsAfterProcess() throws QException + { + List processTraceRecords = QueryAction.execute(ProcessTrace.TABLE_NAME, new QQueryFilter()); + assertEquals(1, processTraceRecords.size()); + + List processTraceSummaryLineRecords = QueryAction.execute(ProcessTraceSummaryLine.TABLE_NAME, new QQueryFilter()); + assertEquals(2, processTraceSummaryLineRecords.size()); + + QRecord okSummaryLineRecord = processTraceSummaryLineRecords.stream().filter(r -> r.getValueString("status").equals(ProcessTraceSummaryStatusEnum.OK.getPossibleValueId())).findFirst().get(); + assertNotNull(okSummaryLineRecord); + assertEquals(2, okSummaryLineRecord.getValueInteger("recordCount")); + + List okRecordInts = QueryAction.execute(ProcessTraceSummaryLineRecordInt.TABLE_NAME, new QQueryFilter(new QFilterCriteria("processTraceSummaryLineId", QCriteriaOperator.EQUALS, okSummaryLineRecord.getValueLong("id")))); + assertEquals(2, okRecordInts.size()); + assertThat(okRecordInts).allMatch(r -> r.getValueInteger("recordId").equals(100) || r.getValueInteger("recordId").equals(101)); + + QRecord errorSummaryLineRecord = processTraceSummaryLineRecords.stream().filter(r -> r.getValueString("status").equals(ProcessTraceSummaryStatusEnum.ERROR.getPossibleValueId())).findFirst().get(); + assertNotNull(errorSummaryLineRecord); + assertEquals(1, errorSummaryLineRecord.getValueInteger("recordCount")); + + List errorRecordInts = QueryAction.execute(ProcessTraceSummaryLineRecordInt.TABLE_NAME, new QQueryFilter(new QFilterCriteria("processTraceSummaryLineId", QCriteriaOperator.EQUALS, errorSummaryLineRecord.getValueLong("id")))); + assertEquals(1, errorRecordInts.size()); + assertEquals(102, errorRecordInts.get(0).getValueInteger("recordId")); + } + +} \ No newline at end of file -- 2.54.0 From 0e556e09e466816b6b3fafccffdf3d3b7e205a93 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Tue, 4 Nov 2025 16:57:39 -0600 Subject: [PATCH 4/5] chore: update version (0.1.3-SNAPSHOT) and otherwise make work also: - downgrade qbit-build-parent - 1.4.0 doesn't seem to exist - update qqq to build for this feature dev - add sonatype repo for qqq snapshots - remove rdbms (not used in here) --- pom.xml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 5a53ab0..5e99d39 100644 --- a/pom.xml +++ b/pom.xml @@ -23,13 +23,13 @@ - 0.1.2 + 0.1.3-SNAPSHOT com.kingsrook qbit-build-parent - 1.4.0 + 1.1.0 com.kingsrook.qbits @@ -68,14 +68,12 @@ HEAD - - com.kingsrook.qqq qqq-bom-pom - 0.27.9 + 0.31.0-expand-query-stats-8b0b96781-20251104.222953-1 pom import @@ -94,10 +92,6 @@ com.kingsrook.qqq qqq-backend-core - - com.kingsrook.qqq - qqq-backend-module-rdbms - org.junit.jupiter @@ -112,4 +106,11 @@ + + + sonatype-snapshots + https://central.sonatype.com/repository/maven-snapshots/ + + + -- 2.54.0 From 3f8d113f6123876e1ab681f018624014cb332702 Mon Sep 17 00:00:00 2001 From: Darin Kelkhoff Date: Wed, 19 Nov 2025 09:13:51 -0600 Subject: [PATCH 5/5] chore: Update qqq version to develop build --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5e99d39..6231f7f 100644 --- a/pom.xml +++ b/pom.xml @@ -73,7 +73,7 @@ com.kingsrook.qqq qqq-bom-pom - 0.31.0-expand-query-stats-8b0b96781-20251104.222953-1 + 0.31.0-20251119.140604-23 pom import -- 2.54.0