feat: Create rows for analyzable tables, Setup card and table help so Analysis Settings is never a mystery
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
james.maesandClaude Opus 5.5 committed
1 parent
69800f755c
commit
04d3c01cf3
15 files changed
+1214
-5
No files matched your search
+297
@@ -0,0 +1,297 @@
|
||||
/*
|
||||
* QQQ - Low-code Application Framework for Engineers.
|
||||
* Copyright (C) 2021-2025. Kingsrook, LLC
|
||||
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
||||
* contact@kingsrook.com
|
||||
* https://github.com/Kingsrook/
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package com.kingsrook.qbits.dataanalysis.processes;
|
||||
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig;
|
||||
import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime;
|
||||
import com.kingsrook.qbits.dataanalysis.engine.EffectiveConfig;
|
||||
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
|
||||
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
|
||||
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
|
||||
import com.kingsrook.qqq.backend.core.context.QContext;
|
||||
import com.kingsrook.qqq.backend.core.exceptions.QException;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
|
||||
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
|
||||
import com.kingsrook.qqq.backend.core.model.data.QRecord;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
|
||||
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractExtractStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
|
||||
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** "Create rows for analyzable tables": a table action on Analysis Settings
|
||||
** that writes one row per analyzable table that has none yet, holding the
|
||||
** settings that already apply to it (code defaults for that table, then the
|
||||
** global defaults). It changes no behavior on its own: it makes the defaults
|
||||
** visible and editable. Existing rows are never touched. Standard streamed
|
||||
** ETL: Review lists what will be added, Result what was added.
|
||||
*******************************************************************************/
|
||||
public final class TableConfigDefaultsProcess
|
||||
{
|
||||
public static final String NAME = "data_analysis_table_config_create_rows";
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Utility class.
|
||||
*******************************************************************************/
|
||||
private TableConfigDefaultsProcess()
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The process: a table-level action (no records need to be selected). Table
|
||||
** permissions (settings read and insert) are enforced by the steps.
|
||||
*******************************************************************************/
|
||||
public static QProcessMetaData process()
|
||||
{
|
||||
Map<String, Serializable> values = new HashMap<>();
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SOURCE_TABLE, DataAnalysisTables.TABLE_CONFIG);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_DESTINATION_TABLE, DataAnalysisTables.TABLE_CONFIG);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_SUPPORTS_FULL_VALIDATION, false);
|
||||
values.put(StreamedETLWithFrontendProcess.FIELD_PREVIEW_MESSAGE,
|
||||
"These rows will be added: one per analyzable table that has none, holding the settings that already apply. Existing rows are not changed.");
|
||||
QProcessMetaData process = StreamedETLWithFrontendProcess.defineProcessMetaData(Extract.class, Transform.class, Load.class, values);
|
||||
process.setName(NAME);
|
||||
process.setLabel("Create rows for analyzable tables");
|
||||
process.setIcon(new QIcon("playlist_add"));
|
||||
process.setTableName(DataAnalysisTables.TABLE_CONFIG);
|
||||
process.setMinInputRecords(0);
|
||||
process.setPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED));
|
||||
return (process);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The tables analysis may be used on: everything but the qbit's own tables,
|
||||
** limited by the includeTables and excludeTables configuration. The same test
|
||||
** the "analyze" table permission uses.
|
||||
*******************************************************************************/
|
||||
public static List<String> analyzableTables(QInstance instance, DataAnalysisQBitConfig config)
|
||||
{
|
||||
return (instance.getTables().values().stream().map(QTableMetaData::getName)
|
||||
.filter(name -> !name.startsWith(DataAnalysisQBitConfig.TABLE_NAME_PREFIX))
|
||||
.filter(name -> config.getIncludeTables().isEmpty() || config.getIncludeTables().contains(name))
|
||||
.filter(name -> !config.getExcludeTables().contains(name)).sorted().toList());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** The rows that are missing: one per analyzable table without a row, with
|
||||
** the effective settings (the same values EffectiveConfigResolver produces).
|
||||
*******************************************************************************/
|
||||
public static List<QRecord> missingRows(QInstance instance, DataAnalysisQBitConfig config) throws QException
|
||||
{
|
||||
Set<String> existing = existingTables();
|
||||
DataAnalysisRuntime.service().getConfigResolver().invalidate();
|
||||
List<QRecord> rows = new ArrayList<>();
|
||||
for(String table : analyzableTables(instance, config))
|
||||
{
|
||||
if(!existing.contains(table))
|
||||
{
|
||||
EffectiveConfig c = DataAnalysisRuntime.service().getConfigResolver().resolve(table);
|
||||
rows.add(new QRecord().withValue("table_name", table).withValue("enabled", c.getEnabled()).withValue("max_rows", c.getMaxRows())
|
||||
.withValue("sample_threshold_rows", c.getSampleThresholdRows()).withValue("sample_rows", c.getSampleRows())
|
||||
.withValue("min_group_size", c.getMinGroupSize()).withValue("cache_ttl_minutes", c.getCacheTtlMinutes())
|
||||
.withValue("analyzable_fields", c.getAnalyzableFields().isEmpty() ? null : String.join(",", new TreeSet<>(c.getAnalyzableFields())))
|
||||
.withValue("excluded_fields", c.getExcludedFields().isEmpty() ? null : String.join(",", new TreeSet<>(c.getExcludedFields()))));
|
||||
}
|
||||
}
|
||||
return (rows);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Table names that already have a settings row (read as the signed-in user).
|
||||
*******************************************************************************/
|
||||
private static Set<String> existingTables() throws QException
|
||||
{
|
||||
QueryInput input = new QueryInput(DataAnalysisTables.TABLE_CONFIG).withFilter(new QQueryFilter());
|
||||
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
Set<String> existing = new HashSet<>();
|
||||
new QueryAction().execute(input).getRecords().forEach(row -> existing.add(row.getValueString("table_name")));
|
||||
return (existing);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Extract: the missing rows, computed (no query of the selected records).
|
||||
*******************************************************************************/
|
||||
public static class Extract extends AbstractExtractStep
|
||||
{
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public Integer doCount(RunBackendStepInput input) throws QException
|
||||
{
|
||||
return (missingRows(QContext.getQInstance(), DataAnalysisRuntime.service().getConfigResolver().getDefaults()).size());
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord row : missingRows(QContext.getQInstance(), DataAnalysisRuntime.service().getConfigResolver().getDefaults()))
|
||||
{
|
||||
getRecordPipe().addRecord(row);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Transform: lists what will be added for the review screen.
|
||||
*******************************************************************************/
|
||||
public static class Transform extends AbstractTransformStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine added = line(Status.OK, "table will get a settings row", "tables will get a settings row",
|
||||
"table got a settings row", "tables got a settings row");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
for(QRecord record : input.getRecords())
|
||||
{
|
||||
added.incrementCountAndAddPrimaryKey(record.getValue("table_name"));
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
added.addSelfToListIfAnyCount(lines);
|
||||
if(lines.isEmpty())
|
||||
{
|
||||
lines.add(new ProcessSummaryLine(Status.INFO, null, "Every analyzable table already has a settings row."));
|
||||
}
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** Load: insert through the standard action with the user's own permissions.
|
||||
*******************************************************************************/
|
||||
public static class Load extends AbstractLoadStep implements ProcessSummaryProviderInterface
|
||||
{
|
||||
private final ProcessSummaryLine created = line(Status.OK, "row was added", "rows were added", "row was added", "rows were added");
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
|
||||
{
|
||||
InsertInput insert = new InsertInput(DataAnalysisTables.TABLE_CONFIG).withRecords(input.getRecords());
|
||||
insert.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
|
||||
for(QRecord record : new InsertAction().execute(insert).getRecords())
|
||||
{
|
||||
if(record.getErrors() != null && !record.getErrors().isEmpty())
|
||||
{
|
||||
throw (new QException("Could not add analysis settings: " + record.getErrors()));
|
||||
}
|
||||
created.incrementCountAndAddPrimaryKey(record.getValue("id"));
|
||||
output.getRecords().add(record);
|
||||
}
|
||||
DataAnalysisRuntime.service().getConfigResolver().invalidate();
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
**
|
||||
*******************************************************************************/
|
||||
@Override
|
||||
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean isForResultScreen)
|
||||
{
|
||||
ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
|
||||
created.addSelfToListIfAnyCount(lines);
|
||||
return (lines);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*******************************************************************************
|
||||
** A summary line with its four messages.
|
||||
*******************************************************************************/
|
||||
private static ProcessSummaryLine line(Status status, String singularFuture, String pluralFuture, String singularPast, String pluralPast)
|
||||
{
|
||||
ProcessSummaryLine line = new ProcessSummaryLine(status, 0, null);
|
||||
line.setSingularFutureMessage(singularFuture);
|
||||
line.setPluralFutureMessage(pluralFuture);
|
||||
line.setSingularPastMessage(singularPast);
|
||||
line.setPluralPastMessage(pluralPast);
|
||||
return (line);
|
||||
}
|
||||
|
||||
}
|
||||
Reference in new issue
Block a user