feat(saved): saved, shared, starred, scheduled and widget analyses, export, SQL handoff

Extension 1.4.0: saved analyses of every kind re-run as the viewer
(fields they cannot analyze are dropped), owner-or-shared READ through
core record locks with owner-only writes (and a database CHECK so a
private row can never lose its read owner), personal stars, schedules
that run as the owner and mail only recipients who could run the analysis
themselves, CSV/XLSX export, the SQL Workbench handoff, an analysis
dashboard widget, and an Analyses home built from standard widgets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
james.maesandClaude Opus 5.5 committed 2026-10-13 21:23:23 -05:00
1 parent 43c7571864
commit d2c9871d98
42 files changed
+3786 -296

No files matched your search

@@ -426,4 +426,59 @@ public class DataAnalysisQBitConfig implements QBitConfig
ownerSessionSupplier = supplier;
return (this);
}
private String messagingProviderName;
private String fromAddress;
private transient java.util.function.Function<String, String> recipientAddressSupplier;
/*******************************************************************************
** Core messaging provider used for scheduled delivery.
*******************************************************************************/
public String getMessagingProviderName()
{
return (messagingProviderName);
}
/*******************************************************************************
** Select the host's configured SMTP or SES provider.
*******************************************************************************/
public DataAnalysisQBitConfig withMessagingProviderName(String value)
{
messagingProviderName = value;
return (this);
}
/*******************************************************************************
** Verified sender address supplied by the host.
*******************************************************************************/
public String getFromAddress()
{
return (fromAddress);
}
/*******************************************************************************
** Set the scheduled email sender.
*******************************************************************************/
public DataAnalysisQBitConfig withFromAddress(String value)
{
fromAddress = value;
return (this);
}
/*******************************************************************************
** Resolve a user id to its current email; never treat recipients as addresses.
*******************************************************************************/
public java.util.function.Function<String, String> getRecipientAddressSupplier()
{
return (recipientAddressSupplier);
}
/*******************************************************************************
** Bind the host directory's user-to-email lookup.
*******************************************************************************/
public DataAnalysisQBitConfig withRecipientAddressSupplier(java.util.function.Function<String, String> value)
{
recipientAddressSupplier = value;
return (this);
}
}
@@ -144,6 +144,13 @@ public class DataAnalysisQBitProducer implements QBitMetaDataProducer<DataAnalys
.withResolveMode(com.kingsrook.qqq.backend.core.alerts.ResolveMode.PRODUCER)
.withSubjectTableName(com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables.WATCH)
.withVisibilityPermission("data_analysis.watch.read"));
com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData.ofOrWithNew(instance).withAlertType(
new com.kingsrook.qqq.backend.core.alerts.QAlertTypeMetaData().withName(com.kingsrook.qbits.dataanalysis.schedule.ScheduleRunner.ALERT_TYPE)
.withLabel("Analysis schedule failed").withSourceName(ARTIFACT_ID).withGroupName("data-analysis")
.withDefaultSeverity(com.kingsrook.qqq.backend.core.alerts.AlertSeverity.ERROR)
.withResolveMode(com.kingsrook.qqq.backend.core.alerts.ResolveMode.PRODUCER)
.withSubjectTableName(com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables.SCHEDULE)
.withVisibilityPermission("data_analysis.saved.read"));
var included = java.util.Set.copyOf(qBitConfig.getIncludeTables());
var excluded = java.util.Set.copyOf(qBitConfig.getExcludeTables());
TableFeaturePermissionsMetaData.ofOrWithNew(instance).withPermission(new TableFeaturePermission()
@@ -30,6 +30,8 @@ import com.kingsrook.qqq.backend.core.model.metadata.QSupplementalInstanceMetaDa
public class DataAnalysisRuntime implements QSupplementalInstanceMetaData
{
private final transient DataAnalysisService service;
private final transient com.kingsrook.qbits.dataanalysis.schedule.ScheduleRunner schedules;
private final transient com.kingsrook.qbits.dataanalysis.saved.SavedAnalyses saved;
private final transient com.kingsrook.qbits.dataanalysis.watch.WatchEvaluator watches;
/*******************************************************************************
@@ -38,6 +40,8 @@ public class DataAnalysisRuntime implements QSupplementalInstanceMetaData
public DataAnalysisRuntime(DataAnalysisQBitConfig config)
{
service = new DataAnalysisService(config);
schedules = new com.kingsrook.qbits.dataanalysis.schedule.ScheduleRunner(config, service);
saved = new com.kingsrook.qbits.dataanalysis.saved.SavedAnalyses(service);
watches = new com.kingsrook.qbits.dataanalysis.watch.WatchEvaluator(config, service);
}
@@ -80,4 +84,20 @@ public class DataAnalysisRuntime implements QSupplementalInstanceMetaData
{
return (((DataAnalysisRuntime) QContext.getQInstance().getSupplementalMetaData("dataAnalysisRuntime")).watches);
}
/*******************************************************************************
** Saved operations share the instance's engine and serialize star toggles.
*******************************************************************************/
public static com.kingsrook.qbits.dataanalysis.saved.SavedAnalyses saved()
{
return (((DataAnalysisRuntime) QContext.getQInstance().getSupplementalMetaData("dataAnalysisRuntime")).saved);
}
/*******************************************************************************
** Shared schedule runner for manual and automatic delivery.
*******************************************************************************/
public static com.kingsrook.qbits.dataanalysis.schedule.ScheduleRunner schedules()
{
return (((DataAnalysisRuntime) QContext.getQInstance().getSupplementalMetaData("dataAnalysisRuntime")).schedules);
}
}
@@ -57,6 +57,14 @@ public final class DataAnalysisSchema
columns.add(" " + field.getBackendName() + " " + ("id".equals(field.getName())
? "INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY" : sqlType(field) + (Boolean.TRUE.equals(field.getIsRequired()) ? " NOT NULL" : "")));
}
if(DataAnalysisTables.SAVED.equals(table.getName()))
{
///////////////////////////////////////////////////////////////////////////////
// a null read-owner projection means "shared": make the database refuse it //
// on a private row, so a row written outside QQQ fails closed, not open //
///////////////////////////////////////////////////////////////////////////////
columns.add(" CONSTRAINT ck_data_analysis_saved_read_owner CHECK (shared = TRUE OR read_owner_user_id IS NOT NULL)");
}
statements.add("CREATE TABLE IF NOT EXISTS " + table.getName() + " (\n" + String.join(",\n", columns) + "\n)");
if(table.getUniqueKeys() != null)
{
@@ -351,4 +351,29 @@ public class DataAnalysisService
return (new com.kingsrook.qbits.dataanalysis.watch.WatchSeries(security, configs.resolve(table), table)
.preview(definition, com.kingsrook.qbits.dataanalysis.watch.WatchRecords.filter(resolved), now));
}
/*******************************************************************************
** Saved definitions drop selected fields using the same FULL access resolver.
*******************************************************************************/
public boolean isAvailableField(String table, String field)
{
try
{
if(field.contains("."))
{
String[] parts = field.split("\\.", 2);
security.joinedField(parts[0], parts[1]);
}
else
{
security.field(table, field);
}
return (true);
}
catch(QException e)
{
return (false);
}
}
}
@@ -0,0 +1,195 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.api;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime;
import com.kingsrook.qbits.dataanalysis.saved.AnalysisExport;
import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.middleware.javalin.executors.AbstractMiddlewareExecutor;
import com.kingsrook.qqq.middleware.javalin.executors.io.AbstractMiddlewareInput;
import com.kingsrook.qqq.middleware.javalin.executors.io.AbstractMiddlewareOutputInterface;
import com.kingsrook.qqq.middleware.javalin.specs.AbstractEndpointSpec;
import com.kingsrook.qqq.middleware.javalin.specs.BasicOperation;
import com.kingsrook.qqq.middleware.javalin.specs.BasicResponse;
import com.kingsrook.qqq.openapi.model.Content;
import com.kingsrook.qqq.openapi.model.HttpMethod;
import com.kingsrook.qqq.openapi.model.In;
import com.kingsrook.qqq.openapi.model.Parameter;
import com.kingsrook.qqq.openapi.model.RequestBody;
import com.kingsrook.qqq.openapi.model.Response;
import com.kingsrook.qqq.openapi.model.Schema;
import com.kingsrook.qqq.openapi.model.Type;
import io.javalin.http.Context;
/*******************************************************************************
** Binary endpoints extend AbstractEndpointSpec directly, preserving core sessions.
*******************************************************************************/
public class AnalysisFileSpec extends AbstractEndpointSpec<AnalysisFileSpec.Input, AnalysisFileSpec.Output, AnalysisFileSpec.Executor>
{
/*******************************************************************************
** POST an arbitrary supported typed request plus CSV/XLSX format.
*******************************************************************************/
@Override
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withPath(DataAnalysisApi.PREFIX + "/export").withHttpMethod(HttpMethod.POST)
.withShortSummary("Export an analysis result").withLongDescription("Execute a typed analysis as the signed-in viewer and download its result table as CSV or XLSX. Requires source READ/Analyze and FULL fields; uses core report formats."));
}
/*******************************************************************************
** Saved export has only id/format parameters; ad hoc export uses a JSON body.
*******************************************************************************/
@Override
public List<Parameter> defineRequestParameters()
{
return (List.of());
}
/*******************************************************************************
** Typed discriminated request plus file format.
*******************************************************************************/
@Override
public RequestBody defineRequestBody()
{
return (new RequestBody().withRequired(true).withContent(Map.of("application/json", new Content()
.withSchema(new Schema().withRef("#/components/schemas/DataAnalysisSavedRequestV1")))));
}
/*******************************************************************************
** Both file representations are declared, without JSON response wrapping.
*******************************************************************************/
@Override
public Map<Integer, Response> defineResponses()
{
Schema file = new Schema().withRef("#/components/schemas/DataAnalysisFileV1");
return (Map.of(200, new Response().withDescription("Analysis result file").withContent(new java.util.TreeMap<>(Map.of("text/csv", new Content().withSchema(file),
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", new Content().withSchema(file))))));
}
/*******************************************************************************
** Read transport data only; execution waits for middleware session setup.
*******************************************************************************/
@Override
public Input buildInput(Context context) throws Exception
{
Input input = new Input();
if(this instanceof Saved)
{
input.id = SavedApi.id(context.pathParam("id"));
input.format = context.queryParam("format");
}
else
{
if(context.contentType() == null || !context.contentType().toLowerCase(java.util.Locale.ROOT).startsWith("application/json"))
{
throw new QBadRequestException("The request body must be JSON.");
}
input.body = SavedDefinition.object(context.body());
input.format = input.body.get("format") instanceof String value ? value : null;
}
AnalysisExport.format(input.format);
return (input);
}
/*******************************************************************************
** Actual file bytes, a stable filename and the matching MIME type.
*******************************************************************************/
@Override
public void handleOutput(Context context, Output output)
{
context.contentType(output.file.mimeType()).header("Content-Disposition", "attachment; filename=\"" + output.file.filename() + "\"")
.header("Cache-Control", "no-store").result(output.file.bytes());
}
/*******************************************************************************
** Saved GET variant uses the same file executor.
*******************************************************************************/
public static class Saved extends AnalysisFileSpec
{
/*******************************************************************************
** Core-visible saved definition is re-run as viewer before export.
*******************************************************************************/
@Override
public BasicOperation defineBasicOperation()
{
return (new BasicOperation().withPath(DataAnalysisApi.PREFIX + "/saved/{id}/export").withHttpMethod(HttpMethod.GET)
.withShortSummary("Export a saved analysis").withLongDescription("Re-run a visible saved analysis as the viewer, applying current source permissions and dropping unavailable fields, then download its result table as CSV or XLSX."));
}
/*******************************************************************************
** Required id and CSV/XLSX query format.
*******************************************************************************/
@Override
public List<Parameter> defineRequestParameters()
{
return (List.of(new Parameter().withName("id").withIn(In.PATH).withRequired(true).withDescription("Visible saved analysis id.").withSchema(new Schema().withType(Type.INTEGER)),
new Parameter().withName("format").withIn(In.QUERY).withRequired(true).withDescription("Result file format.").withSchema(new Schema().withType(Type.STRING).withEnumValues(List.of("csv", "xlsx")))));
}
/*******************************************************************************
** GET has no request body.
*******************************************************************************/
@Override
public RequestBody defineRequestBody()
{
return (null);
}
}
/*******************************************************************************
** Input data only, inherited middleware fields carry the authenticated session.
*******************************************************************************/
public static class Input extends AbstractMiddlewareInput
{
private Integer id;
private String format;
private Map<String, Object> body;
}
/*******************************************************************************
** Binary output stays out of JSON serialization.
*******************************************************************************/
public static class Output implements AbstractMiddlewareOutputInterface
{
private AnalysisExport.File file;
}
/*******************************************************************************
** Invoked after the normal middleware session pipeline.
*******************************************************************************/
public static class Executor extends AbstractMiddlewareExecutor<Input, Output>
{
/*******************************************************************************
** No direct source access outside the existing authorized analysis engine.
*******************************************************************************/
@Override
public void execute(Input input, Output output) throws QException
{
var result = input.id == null ? SavedApi.definition(input.body, null).run(DataAnalysisRuntime.service(), true)
: DataAnalysisRuntime.saved().run(input.id);
output.file = AnalysisExport.file(result, input.format);
}
}
}
@@ -76,8 +76,10 @@ public class DataAnalysisApi
*******************************************************************************/
public static MiddlewareSpecExtension extension()
{
return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.3.0")
return (new MiddlewareSpecExtension().withName("data-analysis").withSemanticVersion("1.4.0")
.withDescription("Profiles, pivots, comparisons, distribution shifts, correlations, scatter samples, crosstabs and anomaly watch previews as the signed-in user.")
.withSpec(SavedApi.Run.class).withSpec(SavedApi.ListSaved.class).withSpec(SavedApi.Star.class).withSpec(SavedApi.Unstar.class)
.withSpec(SavedApi.Sql.class).withSpec(AnalysisFileSpec.class).withSpec(AnalysisFileSpec.Saved.class)
.withSpec(CapabilitiesSpec.class).withSpec(ColumnProfileSpec.class).withSpec(DistinctValuesSpec.class)
.withSpec(CompareSpec.class).withSpec(ShiftSpec.class).withSpec(CorrelationSpec.class).withSpec(ScatterSpec.class).withSpec(CrosstabSpec.class)
.withSpec(WatchPreviewSpec.class).withSpec(PivotSpec.class).withSpec(TableProfileSpec.class).withSpec(DuplicatesSpec.class).withSpec(StartRunSpec.class).withSpec(GetRunSpec.class));
@@ -463,8 +465,7 @@ public class DataAnalysisApi
result.canAnalyze = false;
result.fields = List.of();
result.backendCapabilities = List.of();
result.sqlWorkbenchAvailable = QContext.getQInstance().getApp("sqlWorkbench") != null
|| QContext.getQInstance().getTables().keySet().stream().anyMatch(t -> t.startsWith("sql_workbench_"));
result.sqlWorkbenchAvailable = com.kingsrook.qbits.dataanalysis.saved.SqlHandoff.installedAndAllowed();
try
{
var capabilities = DataAnalysisRuntime.service().capabilities(name);
@@ -116,6 +116,7 @@ public class DataAnalysisSchemas
result.put("DataAnalysisPivotV1", new DataAnalysisPivotV1().toSchema());
result.putAll(RelationshipSchemas.components());
result.putAll(WatchSchemas.components());
result.putAll(SavedSchemas.components());
return (result);
}
@@ -0,0 +1,208 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.api;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime;
import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition;
import com.kingsrook.qbits.dataanalysis.saved.SqlHandoff;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.openapi.model.HttpMethod;
/*******************************************************************************
** Saved execution, stars, visible lists and SQL discovery through the UI API.
*******************************************************************************/
public final class SavedApi
{
/*******************************************************************************
** Namespace only.
*******************************************************************************/
private SavedApi()
{
}
/*******************************************************************************
** Stable integer-id errors, without probing inaccessible records.
*******************************************************************************/
public static Integer id(String value) throws QBadRequestException
{
try
{
return (Integer.valueOf(value));
}
catch(Exception e)
{
throw new QBadRequestException("Saved analysis id must be an integer.");
}
}
/*******************************************************************************
** Accept a discriminated typed request; path table overrides only absent tableName.
*******************************************************************************/
@SuppressWarnings("unchecked")
public static SavedDefinition definition(Map<String, Object> body, String pathTable) throws QException
{
if(!(body.get("kind") instanceof String kind))
{
throw new QBadRequestException("An analysis kind is required.");
}
Map<String, Object> request = new LinkedHashMap<>(body);
request.remove("kind");
request.remove("format");
if(body.containsKey("request"))
{
if(!(body.get("request") instanceof Map<?, ?> map))
{
throw new QBadRequestException("request must be a typed analysis object.");
}
request = new LinkedHashMap<>((Map<String, Object>) map);
}
Object table = pathTable == null ? body.getOrDefault("tableName", request.get("tableName")) : pathTable;
if(!(table instanceof String name))
{
throw new QBadRequestException("tableName is required.");
}
return (SavedDefinition.parse(kind, name, request));
}
/*******************************************************************************
** Run the visible saved definition under the viewer's current source permissions.
*******************************************************************************/
public static class Run extends DataAnalysisApi.ProfileSpec
{
/*******************************************************************************
** Publish assistant-oriented semantics and field-reduction behavior.
*******************************************************************************/
public Run()
{
super("/saved/{id}/run", HttpMethod.POST, "Run a saved analysis",
"Re-run a visible saved analysis as the signed-in viewer. Requires saved READ and source READ/Analyze. Drops unavailable fields with a warning and returns the result typed by kind; inaccessible required axes or predicates produce no result.", null, "SavedResult");
}
/*******************************************************************************
** Core locks determine visibility, not an endpoint-specific owner check.
*******************************************************************************/
@Override
protected Object handle(JsonInput input) throws Exception
{
return (DataAnalysisRuntime.saved().run(id(input.getPathParam("id"))));
}
}
/*******************************************************************************
** Visible saved metadata with this viewer's personal stars.
*******************************************************************************/
public static class ListSaved extends DataAnalysisApi.ProfileSpec
{
/*******************************************************************************
** A discoverable list rather than exposing the owner's cached result JSON.
*******************************************************************************/
public ListSaved()
{
super("/saved", HttpMethod.GET, "List visible saved analyses",
"List core-visible saved analyses with name, kind, source table label, owner, shared state, this viewer's starred flag and last update.", null, "SavedList");
}
/*******************************************************************************
** The list and star lookup both use enforced core queries.
*******************************************************************************/
@Override
protected Object handle(JsonInput input) throws Exception
{
return (Map.of("analyses", DataAnalysisRuntime.saved().list()));
}
}
/*******************************************************************************
** Idempotent star PUT for the signed-in user.
*******************************************************************************/
public static class Star extends DataAnalysisApi.ProfileSpec
{
/*******************************************************************************
** No user-id body is accepted; stars always belong to the caller.
*******************************************************************************/
public Star()
{
super("/saved/{id}/star", HttpMethod.PUT, "Star a saved analysis",
"Star a visible saved analysis for the signed-in user. Idempotent; uses standard saved insert permission and the personal star record lock.", null, "Star");
}
/*******************************************************************************
** Persist through core table CRUD.
*******************************************************************************/
@Override
protected Object handle(JsonInput input) throws Exception
{
return (Map.of("starred", DataAnalysisRuntime.saved().star(id(input.getPathParam("id")), true)));
}
}
/*******************************************************************************
** Idempotent star DELETE for the signed-in user.
*******************************************************************************/
public static class Unstar extends DataAnalysisApi.ProfileSpec
{
/*******************************************************************************
** Removal never targets another user's star.
*******************************************************************************/
public Unstar()
{
super("/saved/{id}/star", HttpMethod.DELETE, "Unstar a saved analysis",
"Remove the signed-in user's star on a visible saved analysis. Idempotent; uses standard saved delete permission and owner locks.", null, "Star");
}
/*******************************************************************************
** Persist through core table CRUD.
*******************************************************************************/
@Override
protected Object handle(JsonInput input) throws Exception
{
return (Map.of("starred", DataAnalysisRuntime.saved().star(id(input.getPathParam("id")), false)));
}
}
/*******************************************************************************
** Actual backend aggregate SQL, captured only during this request.
*******************************************************************************/
public static class Sql extends DataAnalysisApi.ProfileSpec
{
/*******************************************************************************
** The assistant gets a reading aid, never an alternative execution endpoint.
*******************************************************************************/
public Sql()
{
super("/tables/{table}/sql", HttpMethod.POST, "Read an analysis aggregate SQL statement",
"Execute a typed analysis as the viewer and capture the main RDBMS aggregate SQL from core QueryStat. Requires installed SQL Workbench and its access permission. Non-RDBMS backends report unavailable; SQL is for reading and record security remains enforced at runtime.", "SavedRequest", "Sql");
}
/*******************************************************************************
** Requests pass through the same parser and analysis authorization.
*******************************************************************************/
@Override
protected Object handle(JsonInput input) throws Exception
{
return (SqlHandoff.execute(definition(input.getBody(), input.getPathParam("table")), DataAnalysisRuntime.service()));
}
}
}
@@ -0,0 +1,99 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.api;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import com.kingsrook.qqq.openapi.model.Schema;
import com.kingsrook.qqq.openapi.model.Type;
/*******************************************************************************
** Versioned saved envelopes reuse all seven existing typed request/result schemas.
*******************************************************************************/
public final class SavedSchemas
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private SavedSchemas()
{
}
/*******************************************************************************
** JSON Schema union supported by the standard OpenAPI serializer.
*******************************************************************************/
public static class RequestUnion extends Schema
{
/*******************************************************************************
** All supported typed analysis requests, selected by the sibling kind.
*******************************************************************************/
public List<Schema> getAnyOf()
{
return (List.of("ColumnProfile", "TableProfile", "Pivot", "Compare", "Correlation", "Crosstab", "Shift").stream()
.map(name -> ref(name + "Request")).toList());
}
}
/*******************************************************************************
** Every new endpoint's request and response is discoverable by clients.
*******************************************************************************/
public static Map<String, Schema> components()
{
Map<String, Schema> result = new LinkedHashMap<>();
Map<String, Schema> fields = new LinkedHashMap<>();
fields.put("kind", new Schema().withType(Type.STRING).withEnumValues(List.of("COLUMN_PROFILE", "TABLE_PROFILE", "PIVOT", "COMPARE", "CORRELATION", "CROSSTAB", "SHIFT")));
fields.put("tableName", new Schema().withType(Type.STRING).withDescription("Source QQQ table name; required for ad hoc export, supplied by the path for SQL."));
fields.put("request", new RequestUnion().withDescription("Typed request matching kind. Field names, filters and dimensions use the existing analysis request contracts."));
fields.put("format", new Schema().withType(Type.STRING).withEnumValues(List.of("csv", "xlsx")).withDescription("Required for export; ignored for SQL."));
result.put("DataAnalysisSavedRequestV1", new DataAnalysisSchemas.RequiredSchema(List.of("kind", "request")).withType(Type.OBJECT).withProperties(fields));
Map<String, Schema> response = new LinkedHashMap<>();
response.put("kind", fields.get("kind"));
response.put("warnings", new Schema().withType(Type.ARRAY).withItems(new Schema().withType(Type.STRING)));
new java.util.TreeMap<>(Map.of("columnProfile", "ColumnProfile", "tableProfile", "TableProfile", "pivot", "Pivot", "compare", "Compare",
"correlation", "Correlation", "crosstab", "Crosstab", "shift", "Shift")).forEach((key, value) -> response.put(key, ref(value).withNullable(true)));
result.put("DataAnalysisSavedResultV1", new Schema().withType(Type.OBJECT).withProperties(response));
Map<String, Schema> entry = new LinkedHashMap<>();
entry.put("id", new Schema().withType(Type.INTEGER));
for(String name : List.of("name", "kind", "tableName", "tableLabel", "owner", "updated"))
{
entry.put(name, new Schema().withType(Type.STRING));
}
entry.put("updated", new Schema().withType(Type.STRING).withFormat("date-time").withNullable(true));
entry.put("shared", new Schema().withType(Type.BOOLEAN));
entry.put("starred", new Schema().withType(Type.BOOLEAN));
result.put("DataAnalysisSavedEntryV1", new Schema().withType(Type.OBJECT).withProperties(entry));
result.put("DataAnalysisSavedListV1", new Schema().withType(Type.OBJECT).withProperties(new java.util.TreeMap<>(Map.of("analyses", new Schema().withType(Type.ARRAY).withItems(ref("SavedEntry"))))));
result.put("DataAnalysisStarV1", new Schema().withType(Type.OBJECT).withProperties(new java.util.TreeMap<>(Map.of("starred", new Schema().withType(Type.BOOLEAN)))));
result.put("DataAnalysisSqlV1", new Schema().withType(Type.OBJECT).withProperties(new java.util.TreeMap<>(Map.of("available", new Schema().withType(Type.BOOLEAN),
"sql", new Schema().withType(Type.STRING).withNullable(true), "note", new Schema().withType(Type.STRING)))));
result.put("DataAnalysisFileV1", new Schema().withType(Type.STRING).withFormat("binary"));
return (result);
}
/*******************************************************************************
** Reuse the extension's stable schema prefix.
*******************************************************************************/
private static Schema ref(String name)
{
return (new Schema().withRef("#/components/schemas/DataAnalysis" + name + "V1"));
}
}
@@ -20,35 +20,35 @@
package com.kingsrook.qbits.dataanalysis.app;
import java.time.Instant;
import java.util.Arrays;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.stream.Collectors;
import java.util.Map;
import java.util.Objects;
import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig;
import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting;
import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime;
import com.kingsrook.qbits.dataanalysis.engine.DrillFilters;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.saved.SavedAnalyses;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.QWidgetData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
@@ -58,35 +58,49 @@ import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData;
/*******************************************************************************
** Analyses app home: real counts under the current user's permissions and locks.
** Standard QQQ statistics and narrow table widgets, all queried as the viewer.
*******************************************************************************/
public class DataAnalysisHome extends AbstractWidgetRenderer
{
public static final String HOME_WIDGET = "dataAnalysisHome";
public static final String STARRED = "dataAnalysisStarred";
public static final String FIRING = "dataAnalysisFiring";
public static final String RECENT = "dataAnalysisRecent";
/*******************************************************************************
** The home itself has no data privilege; each section checks its table READ.
** The existing home name now supplies standard multi-statistics data.
*******************************************************************************/
public static QWidgetMetaData widget()
{
return (new QWidgetMetaData().withName(HOME_WIDGET).withLabel("Overview").withType(WidgetType.HTML.getType())
.withCodeReference(new QCodeReference(DataAnalysisHome.class)).withGridColumns(12).withIsCard(false)
return (widget(HOME_WIDGET, "Analysis overview", WidgetType.MULTI_STATISTICS));
}
/*******************************************************************************
** Full-width tables have three or four columns and honest empty states.
*******************************************************************************/
public static List<QWidgetMetaData> widgets()
{
return (List.of(widget(), widget(STARRED, "Starred analyses", WidgetType.TABLE), widget(FIRING, "Firing watches", WidgetType.TABLE), widget(RECENT, "My recent runs", WidgetType.TABLE)));
}
/*******************************************************************************
** The widget itself grants no privilege; each data section checks table READ.
*******************************************************************************/
private static QWidgetMetaData widget(String name, String label, WidgetType type)
{
return (new QWidgetMetaData().withName(name).withLabel(label).withType(type.getType()).withGridColumns(12).withIsCard(true)
.withShowReloadButton(true).withCodeReference(new QCodeReference(DataAnalysisHome.class))
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)));
}
/*******************************************************************************
** Only main tables appear in navigation; schedules and stars are record children.
** Main tables remain navigation; schedules and stars remain child lists.
*******************************************************************************/
public static QAppMetaData app(DataAnalysisQBitConfig config, List<QTableMetaData> tables)
{
QAppMetaData app = new QAppMetaData().withName(config.getAppName()).withLabel(config.getAppLabel())
.withIcon(new QIcon("insights")).withWidgets(List.of(HOME_WIDGET))
.withIcon(new QIcon("insights")).withWidgets(widgets().stream().map(QWidgetMetaData::getName).toList())
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED))
.withSection(new QAppSection().withName("analyses").withLabel("Analyses").withIcon(new QIcon("insights")).withTables(DataAnalysisTables.NAV_TABLES));
app.setParentAppName(config.getParentAppName());
@@ -100,177 +114,172 @@ public class DataAnalysisHome extends AbstractWidgetRenderer
/*******************************************************************************
** RawHTML is carried as a JSON string by the HTML widget, not a text/html response.
** Entities keep static separators independent of any host charset handling.
** Dispatch data by registered metadata name, never by an arbitrary table input.
*******************************************************************************/
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
return (new RenderWidgetOutput(new RawHTML(null, render(Instant.now()))));
return (new RenderWidgetOutput(render(input.getWidgetMetaData().getName(), Instant.now())));
}
/*******************************************************************************
** Counts and table rows share core-enforced permissions and record locks.
*******************************************************************************/
public static QWidgetData render(String name, Instant now) throws QException
{
if(HOME_WIDGET.equals(name))
{
return (statistics(now));
}
if(STARRED.equals(name))
{
TableData data = table("Starred analyses", DataAnalysisTables.SAVED, "No starred analyses yet. Star a saved analysis to see it here.", "Name", "Table", "Kind");
if(canRead(DataAnalysisTables.SAVED))
{
for(var entry : DataAnalysisRuntime.saved().list().stream().filter(SavedAnalyses.Entry::starred).toList())
{
data.getRows().add(values(link(DataAnalysisTables.SAVED, entry.id(), entry.name()), entry.tableLabel(), entry.kind()));
}
}
return (data);
}
String source = FIRING.equals(name) ? DataAnalysisTables.WATCH : DataAnalysisTables.RUN;
TableData data = FIRING.equals(name)
? table("Firing watches", source, "No firing watches.", "Watch", "Table", "Last evaluated", "Message")
: table("My recent runs", source, "No analysis runs yet.", "Analysis", "Table", "Status", "Started");
if(!canRead(source))
{
return (data);
}
QQueryFilter filter = FIRING.equals(name) ? equal("state", "FIRING")
: equal("user_id", SavedAnalyses.user()).withCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now);
filter.withOrderBy(new QFilterOrderBy(FIRING.equals(name) ? "last_evaluated_at" : "started_at", false)).withLimit(10);
for(var row : SavedAnalyses.query(source, filter))
{
String tableName = row.getValueString("table_name");
var metadata = QContext.getQInstance().getTable(tableName);
String label = metadata == null ? Objects.toString(tableName, "Saved analysis") : metadata.getLabel();
data.getRows().add(FIRING.equals(name)
? values(link(source, row.getValue("id"), row.getValueString("detector")), label, row.getValue("last_evaluated_at"), row.getValue("last_message"))
: values(link(source, row.getValue("id"), row.getValueString("operation")), label, row.getValue("status"), row.getValue("started_at")));
}
return (data);
}
/*******************************************************************************
** Counts are evaluated at one instant; no system session or synthetic numbers.
** One standard group per subject, with real counts or an unavailable explanation.
*******************************************************************************/
public static String render(Instant now) throws QException
public static MultiStatisticsData statistics(Instant now) throws QException
{
StringBuilder html = new StringBuilder("<div class=\"data-analysis-home\" style=\"display:grid;gap:1rem;min-width:0;overflow-wrap:anywhere\">");
String userId = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? null : QContext.getQSession().getUser().getIdReference();
html.append("<section><h3>Saved analyses</h3>");
if(userId == null)
List<MultiStatisticsData.StatisticsGroupData> groups = new ArrayList<>();
var saved = group("Saved analyses", DataAnalysisTables.SAVED);
if(canRead(DataAnalysisTables.SAVED))
{
html.append("<p>Sign in to see your saved analyses and stars.</p>");
List<SavedAnalyses.Entry> entries = DataAnalysisRuntime.saved().list();
String user = SavedAnalyses.user();
saved.withStatisticList(List.of(stat("Mine", (int) entries.stream().filter(e -> e.owner().equals(user)).count(), DataAnalysisTables.SAVED),
stat("Shared", (int) entries.stream().filter(e -> Boolean.TRUE.equals(e.shared())).count(), DataAnalysisTables.SAVED),
stat("Starred", (int) entries.stream().filter(SavedAnalyses.Entry::starred).count(), DataAnalysisTables.SAVED)));
}
else if(canRead(DataAnalysisTables.SAVED))
{
int mine = count(DataAnalysisTables.SAVED, equal("owner_user_id", userId));
int shared = count(DataAnalysisTables.SAVED, equal("shared", true));
html.append("<p>Mine: ").append(AnalysisFormatting.number(mine, 0)).append(" &middot; Shared (visible to you): ").append(AnalysisFormatting.number(shared, 0)).append("</p>");
if(canRead(DataAnalysisTables.SAVED_STAR))
{
CountInput stars = new CountInput(DataAnalysisTables.SAVED).withIncludeDistinctCount(true)
.withQueryJoin(new QueryJoin(QContext.getQInstance().getJoin(DataAnalysisTables.joinName(DataAnalysisTables.SAVED_STAR))).withType(QueryJoin.Type.INNER))
.withFilter(equal(DataAnalysisTables.SAVED_STAR + ".user_id", userId));
stars.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
int starred = new CountAction().execute(stars).getDistinctCount();
html.append("<p>Starred by me: ").append(AnalysisFormatting.number(starred, 0)).append("</p>");
}
if(mine == 0 && shared == 0)
{
html.append("<p>No saved analyses yet. Open any table and choose Analyze on a column header to explore its data.</p>");
}
link(html, DataAnalysisTables.SAVED);
}
else
{
html.append("<p>Saved analyses are unavailable: you need data_analysis.saved.read.</p>");
}
html.append("</section><section><h3>Watches</h3>");
groups.add(saved);
var watches = group("Watches", DataAnalysisTables.WATCH);
if(canRead(DataAnalysisTables.WATCH))
{
int total = count(DataAnalysisTables.WATCH, new QQueryFilter());
int enabled = count(DataAnalysisTables.WATCH, equal("enabled", true));
int firing = count(DataAnalysisTables.WATCH, equal("state", "FIRING"));
html.append("<p>Total: ").append(AnalysisFormatting.number(total, 0)).append(" &middot; Enabled: ").append(AnalysisFormatting.number(enabled, 0)).append(" &middot; Firing: ").append(AnalysisFormatting.number(firing, 0)).append("</p>");
if(total == 0)
{
html.append("<p>No watches yet.</p>");
}
QueryInput watches = new QueryInput(DataAnalysisTables.WATCH).withShouldFetchHeavyFields(true)
.withFilter(new QQueryFilter().withOrderBy(new QFilterOrderBy("state", true)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(5));
watches.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
List<QRecord> recent = new QueryAction().execute(watches).getRecords();
if(!recent.isEmpty())
{
html.append("<ul>");
for(QRecord watch : recent)
{
String path = QContext.getQInstance().getTablePath(DataAnalysisTables.WATCH) + "/" + watch.getValueInteger("id");
html.append("<li><a href=\"").append(escape(path)).append("\">")
.append(escape(words(watch.getValueString("state")))).append(" &middot; ")
.append(escape(words(watch.getValueString("detector")))).append("</a>");
if(watch.getValueString("last_message") != null)
{
html.append(" — ").append(escape(watch.getValueString("last_message")));
}
html.append("</li>");
}
html.append("</ul>");
}
link(html, DataAnalysisTables.WATCH);
watches.withStatisticList(List.of(stat("Enabled", count(DataAnalysisTables.WATCH, equal("enabled", true)), DataAnalysisTables.WATCH),
stat("Firing", count(DataAnalysisTables.WATCH, equal("state", "FIRING")), DataAnalysisTables.WATCH)));
}
else
groups.add(watches);
var runs = group("My runs", DataAnalysisTables.RUN);
if(canRead(DataAnalysisTables.RUN))
{
html.append("<p>Watches are unavailable: you need data_analysis.watch.read.</p>");
runs.withStatisticList(List.of(stat("Last 7 days", count(DataAnalysisTables.RUN, equal("user_id", SavedAnalyses.user())
.withCriteria("started_at", QCriteriaOperator.GREATER_THAN_OR_EQUALS, now.minus(7, ChronoUnit.DAYS))
.withCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)), DataAnalysisTables.RUN)));
}
html.append("</section><section><h3>Recent analysis runs</h3>");
if(userId == null)
{
html.append("<p>Sign in to see your recent analysis runs.</p>");
}
else if(canRead(DataAnalysisTables.RUN))
{
QueryInput query = new QueryInput(DataAnalysisTables.RUN).withFilter(equal("user_id", userId)
.withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now))
.withOrderBy(new QFilterOrderBy("started_at", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(5));
query.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
List<QRecord> runs = new QueryAction().execute(query).getRecords();
if(runs.isEmpty())
{
html.append("<p>No analysis runs yet.</p>");
}
else
{
html.append("<ul class=\"data-analysis-runs\">");
for(QRecord run : runs)
{
QTableMetaData table = QContext.getQInstance().getTable(run.getValueString("table_name"));
String tableLabel = table == null ? run.getValueString("table_name") : table.getLabel();
String path = QContext.getQInstance().getTablePath(DataAnalysisTables.RUN) + "/" + run.getValueInteger("id");
html.append("<li><a href=\"").append(escape(path)).append("\">").append(escape(tableLabel)).append(" &middot; ")
.append(escape(words(run.getValueString("operation")))).append("</a><dl class=\"data-analysis-run-details\"><dt>Status</dt><dd>")
.append(escape(words(run.getValueString("status")))).append("</dd><dt>Finished</dt><dd>")
.append(run.getValue("finished_at") == null ? "Not finished" : escape(AnalysisFormatting.instant(run.getValueInstant("finished_at"))))
.append("</dd></dl></li>");
}
html.append("</ul>");
}
link(html, DataAnalysisTables.RUN);
}
else
{
html.append("<p>Run history is unavailable: you need data_analysis.run.read.</p>");
}
return (html.append("</section></div>").toString());
groups.add(runs);
return (new MultiStatisticsData("Analyses", groups));
}
/*******************************************************************************
** Explicit permission checks avoid presenting denied sections as empty data.
** Empty/denied data are different messages, not synthetic zeroes.
*******************************************************************************/
private static boolean canRead(String table)
private static MultiStatisticsData.StatisticsGroupData group(String label, String table)
{
return (PermissionsHelper.hasTablePermission(new CountInput(table), table, TablePermissionSubType.READ));
return (new MultiStatisticsData.StatisticsGroupData().withHeader(label).withStatisticList(List.of())
.withSubheader(canRead(table) ? "Visible to you" : "Unavailable: READ permission required."));
}
/*******************************************************************************
** Equality filter for one count.
** Link every count to its actual table path.
*******************************************************************************/
private static QQueryFilter equal(String field, java.io.Serializable value)
private static MultiStatisticsData.StatisticsGroupData.Statistic stat(String label, int count, String table) throws QException
{
return (new QQueryFilter().withCriteria(new QFilterCriteria(field, QCriteriaOperator.EQUALS, value)));
return (new MultiStatisticsData.StatisticsGroupData.Statistic(label, count, QContext.getQInstance().getTablePath(table)));
}
/*******************************************************************************
** Stored operation and status keys are displayed as ordinary words.
** Standard table cells support escaped record links; other columns are plain text.
*******************************************************************************/
private static String words(String value)
private static TableData table(String label, String table, String empty, String... headers) throws QException
{
return (value == null ? "Unknown" : Arrays.stream(value.toLowerCase(Locale.ROOT).split("_"))
.filter(part -> !part.isEmpty()).map(part -> Character.toUpperCase(part.charAt(0)) + part.substring(1)).collect(Collectors.joining(" ")));
List<TableData.Column> columns = new ArrayList<>();
for(int i = 0; i < headers.length; i++)
{
columns.add(new TableData.Column(i == 0 ? "html" : "text", headers[i], "c" + i, null, "left"));
}
return (new TableData(label, columns, new ArrayList<>()).withNoRowsFoundHTML(canRead(table) ? empty : "Unavailable: READ permission required.")
.withLinkText("View all").withLinkURL(QContext.getQInstance().getTablePath(table)).withRowsPerPage(10));
}
/*******************************************************************************
** Preserve real values while avoiding null map entries and executable HTML.
*******************************************************************************/
private static Map<String, Object> values(Object... values)
{
Map<String, Object> row = new LinkedHashMap<>();
for(int i = 0; i < values.length; i++)
{
row.put("c" + i, Objects.toString(values[i], ""));
}
return (row);
}
/*******************************************************************************
** Metadata and persisted values must never become executable widget markup.
** Standard table HTML cell contains only an escaped link and escaped text.
*******************************************************************************/
private static String link(String table, Object id, String label) throws QException
{
return ("<a href=\"" + escape(QContext.getQInstance().getTablePath(table) + "/" + id) + "\">" + escape(label) + "</a>");
}
/*******************************************************************************
** Metadata labels and stored names are never trusted markup.
*******************************************************************************/
private static String escape(String value)
{
return (value == null ? "Unknown" : value.replace("&", "&amp;").replace("\"", "&quot;").replace("<", "&lt;").replace(">", "&gt;").replace("'", "&#39;"));
return (Objects.toString(value, "").replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;").replace("'", "&#39;"));
}
/*******************************************************************************
** Explicit checks distinguish denied sections from empty sections.
*******************************************************************************/
private static boolean canRead(String table)
{
CountInput input = new CountInput(table);
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
return (PermissionsHelper.hasTablePermission(input, table, TablePermissionSubType.READ));
}
/*******************************************************************************
** CountAction enforces table permission and record locks using QContext's session.
** Simple filter; record security is applied exclusively by core.
*******************************************************************************/
private static QQueryFilter equal(String field, java.io.Serializable value)
{
return (DrillFilters.condition(field, QCriteriaOperator.EQUALS, value));
}
/*******************************************************************************
** Counts use the viewer's session and explicitly enforce permissions.
*******************************************************************************/
private static int count(String table, QQueryFilter filter) throws QException
{
@@ -278,16 +287,4 @@ public class DataAnalysisHome extends AbstractWidgetRenderer
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
return (new CountAction().execute(input).getCount());
}
/*******************************************************************************
** Use the instance's actual app path, including custom names and parent apps.
*******************************************************************************/
private static void link(StringBuilder html, String table) throws QException
{
String path = QContext.getQInstance().getTablePath(table);
html.append("<a href=\"").append(escape(path))
.append("\">View all</a>");
}
}
@@ -0,0 +1,157 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.app;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.result.Chart;
import com.kingsrook.qbits.dataanalysis.result.Metric;
import com.kingsrook.qbits.dataanalysis.saved.AnalysisExport;
import com.kingsrook.qbits.dataanalysis.watch.WatchRecords;
import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartSubheaderData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
/*******************************************************************************
** Saved analysis rendered through Next's existing standard chart widget.
*******************************************************************************/
public class SavedAnalysisWidget extends AbstractWidgetRenderer
{
public static final String NAME = "DATA_ANALYSIS_SAVED";
/*******************************************************************************
** Hosts clone this metadata and set savedId to place an analysis on a dashboard.
*******************************************************************************/
public static QWidgetMetaData widget()
{
return (new QWidgetMetaData().withName(NAME).withLabel("Saved analysis").withType(WidgetType.BAR_CHART.getType())
.withGridColumns(12).withIsCard(true).withCodeReference(new QCodeReference(SavedAnalysisWidget.class))
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED)));
}
/*******************************************************************************
** Values come from a fresh viewer run; no stored owner result is ever reused.
*******************************************************************************/
@Override
public RenderWidgetOutput render(RenderWidgetInput input) throws QException
{
var defaults = ((QWidgetMetaData) input.getWidgetMetaData()).getDefaultValues();
Object value = defaults == null ? null : defaults.get("savedId");
if(value == null)
{
return (new RenderWidgetOutput(new ChartData("Saved analysis", "Choose a saved analysis for this widget.", "Analysis", List.of(), List.of())));
}
Integer id;
try
{
id = Integer.valueOf(value.toString());
}
catch(NumberFormatException e)
{
throw new QBadRequestException("savedId must be an integer.");
}
var saved = WatchRecords.get(DataAnalysisTables.SAVED, id);
var result = DataAnalysisRuntime.saved().run(id);
List<Metric> metrics = AnalysisExport.headlines(result);
Chart chart = null;
if(result.columnProfile() != null)
{
chart = result.columnProfile().getChart();
}
else if(result.tableProfile() != null)
{
chart = result.tableProfile().getHeatmap();
}
else if(result.correlation() != null)
{
chart = result.correlation().getChart();
}
else if(result.shift() != null)
{
chart = result.shift().getChart();
}
else if(result.compare() != null && !result.compare().getFields().isEmpty())
{
chart = result.compare().getFields().getFirst().getChart();
}
String description = metrics.stream().filter(Objects::nonNull).map(m -> m.getLabel() + ": " + AnalysisExport.value(m))
.collect(java.util.stream.Collectors.joining("; "));
if(!result.warnings().isEmpty())
{
description += " " + String.join("; ", result.warnings());
}
ChartData data = new ChartData().withTitle(saved.getValueString("name")).withDescription(description).withHeight(280);
String path = QContext.getQInstance().getTablePath(DataAnalysisTables.SAVED) + "/" + id;
Number mainNumber = metrics.stream().filter(m -> !Boolean.TRUE.equals(m.getSuppressed()) && m.getValue() instanceof Number)
.map(m -> (Number) m.getValue()).findFirst().orElse(null);
data.withChartSubheaderData(new ChartSubheaderData().withMainNumber(mainNumber).withMainNumberUrl(path));
List<String> labels = new ArrayList<>();
List<ChartData.Data.Dataset> datasets = new ArrayList<>();
if(chart != null && !chart.getSeries().isEmpty())
{
chart.getSeries().getFirst().getPoints().forEach(p -> labels.add(Objects.toString(p.getLabel(), Objects.toString(p.getX(), ""))));
for(var series : chart.getSeries())
{
datasets.add(new ChartData.Data.Dataset().withLabel(series.getName())
.withData(series.getPoints().stream().map(p -> (Number) p.getY()).toList()));
}
}
else if(result.pivot() != null)
{
List<Number> numbers = new ArrayList<>();
for(var cell : result.pivot().getCells())
{
labels.add(java.util.stream.Stream.concat(result.pivot().getRowHeaders().get(cell.getRowIndex()).stream(),
result.pivot().getColumnHeaders().get(cell.getColumnIndex()).stream()).map(p -> p.getLabel()).collect(java.util.stream.Collectors.joining(" / ")));
Metric metric = cell.getValues().getFirst();
numbers.add(!Boolean.TRUE.equals(metric.getSuppressed()) && metric.getValue() instanceof Number n ? n : null);
}
datasets.add(new ChartData.Data.Dataset().withLabel(result.pivot().getMeasures().getFirst().getLabel()).withData(numbers));
}
else if(result.crosstab() != null)
{
List<Number> numbers = new ArrayList<>();
for(var cell : result.crosstab().getCells())
{
labels.add(result.crosstab().getRows().get(cell.getRowIndex()).getLabel() + " / " + result.crosstab().getColumns().get(cell.getColumnIndex()).getLabel());
Metric metric = cell.getCount();
numbers.add(!Boolean.TRUE.equals(metric.getSuppressed()) && metric.getValue() instanceof Number n ? n : null);
}
datasets.add(new ChartData.Data.Dataset().withLabel("Count").withData(numbers));
}
data.withChartData(new ChartData.Data().withLabels(labels).withUrls(labels.stream().map(label -> path).toList()).withDatasets(datasets));
data.withFooterHTML("<a href=\"" + path.replace("&", "&amp;").replace("\"", "&quot;") + "\">View saved analysis</a>");
return (new RenderWidgetOutput(data));
}
}
@@ -41,7 +41,6 @@ import com.kingsrook.qbits.dataanalysis.result.PivotRequest;
import com.kingsrook.qbits.dataanalysis.result.PivotResult;
import com.kingsrook.qbits.dataanalysis.result.ValueType;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
@@ -617,7 +616,7 @@ public class PivotExplorer
groups.forEach(input::withGroupBy);
new LinkedHashSet<>(aggregates).forEach(input::withAggregate);
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
var output = new AggregateAction().execute(input);
var output = com.kingsrook.qbits.dataanalysis.saved.SqlHandoff.aggregate(input);
partial |= output.getTruncated();
return (limit == null ? output.getResults() : output.getResults().stream().sorted(java.util.Comparator.comparingLong(this::count).reversed()
.thenComparing(r -> String.valueOf(r.getGroupByValues().values()))).limit(limit).toList());
@@ -33,7 +33,6 @@ import com.kingsrook.qbits.dataanalysis.result.Significance;
import com.kingsrook.qbits.dataanalysis.result.ValueType;
import com.kingsrook.qbits.dataanalysis.stats.Statistics;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.Aggregate;
@@ -79,7 +78,7 @@ public class RelationshipSupport extends StandInAnalysisBackend
{
input.withFilter(DrillFilters.restoreTypes(table, input.getFilter())).withAllowStreamedFallback(true).withStreamedRowLimit(config.getMaxRows());
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
var output = new AggregateAction().execute(input);
var output = com.kingsrook.qbits.dataanalysis.saved.SqlHandoff.aggregate(input);
if(Boolean.TRUE.equals(output.getTruncated()))
{
throw new QBadRequestException("Analysis exceeds the streamed row limit; narrow the filter.");
@@ -38,7 +38,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.function.Function;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.actions.tables.CountAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
@@ -171,7 +170,7 @@ public class StandInAnalysisBackend implements AnalysisBackend
restored.setOrderBys(input.getFilter().getOrderBys());
input.setFilter(restored);
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
return (new AggregateAction().execute(input).getResults());
return (com.kingsrook.qbits.dataanalysis.saved.SqlHandoff.aggregate(input).getResults());
}
@@ -66,7 +66,8 @@ public class DataAnalysisMetaDataProducer extends QBitComponentMetaDataProducer<
.filter(qbit -> DataAnalysisQBitProducer.ARTIFACT_ID.equals(qbit.getArtifactId())).findFirst().orElseThrow().getName());
output.add(table);
}
output.add(DataAnalysisHome.widget());
DataAnalysisHome.widgets().forEach(output::add);
output.add(com.kingsrook.qbits.dataanalysis.app.SavedAnalysisWidget.widget());
output.add(DataAnalysisHome.app(getQBitConfig(), tables));
return (output);
}
@@ -0,0 +1,93 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.metadata;
import java.util.List;
import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.processes.ScheduleActions;
import com.kingsrook.qqq.backend.core.model.metadata.MetaDataProducerMultiOutput;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QIcon;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.PermissionLevel;
import com.kingsrook.qqq.backend.core.model.metadata.permissions.QPermissionRules;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QBackendStepMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.processes.QProcessMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.qbits.QBitComponentMetaDataProducer;
import com.kingsrook.qqq.backend.core.model.metadata.scheduleing.QScheduleMetaData;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ExtractViaQueryStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.StreamedETLWithFrontendProcess;
/*******************************************************************************
** Watch record actions and one hidden scheduled process, never app children.
*******************************************************************************/
public class ScheduleProcesses extends QBitComponentMetaDataProducer<MetaDataProducerMultiOutput, DataAnalysisQBitConfig>
{
/*******************************************************************************
** Build after the watch table has been registered.
*******************************************************************************/
@Override
public int getSortOrder()
{
return (200);
}
/*******************************************************************************
** Component scanning also encounters unconfigured producers.
*******************************************************************************/
@Override
public boolean isEnabled()
{
return (getQBitConfig() != null);
}
/*******************************************************************************
** Ordinary core actions provide selected records, Review, and Result screens.
*******************************************************************************/
@Override
public MetaDataProducerMultiOutput produce(QInstance instance)
{
MetaDataProducerMultiOutput output = new MetaDataProducerMultiOutput();
for(String name : List.of(ScheduleActions.SEND))
{
String label = "Send now";
QProcessMetaData process = StreamedETLWithFrontendProcess.processMetaDataBuilder()
.withName(name).withLabel(label).withIcon(new QIcon("notifications_active"))
.withTableName(DataAnalysisTables.SCHEDULE).withSourceTable(DataAnalysisTables.SCHEDULE).withDestinationTable(DataAnalysisTables.SCHEDULE)
.withExtractStepClass(ExtractViaQueryStep.class).withTransformStepClass(ScheduleActions.Review.class).withLoadStepClass(ScheduleActions.Result.class)
.withSupportsFullValidation(true).withDoFullValidation(true).withMinInputRecords(1)
.withPreviewMessage("Review the selected schedules.").getProcessMetaData();
// Table READ/EDIT and owner locks are enforced by extraction and both steps.
process.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.NOT_PROTECTED));
output.add(process);
}
QProcessMetaData scheduled = new QProcessMetaData().withName(ScheduleActions.SCHEDULED).withLabel("Run analysis schedules").withIsHidden(true)
.withPermissionRules(new QPermissionRules().withLevel(PermissionLevel.HAS_ACCESS_PERMISSION).withPermissionBaseName(ScheduleActions.SCHEDULED))
.withStep(new QBackendStepMetaData().withName("run").withCode(new QCodeReference(ScheduleActions.Scheduled.class)));
if(getQBitConfig().getSchedulerName() != null)
{
scheduled.withSchedule(new QScheduleMetaData().withSchedulerName(getQBitConfig().getSchedulerName()).withRepeatSeconds(300).withInitialDelaySeconds(300));
}
output.add(scheduled);
return (output);
}
}
@@ -239,8 +239,15 @@ public final class DataAnalysisTables
.withField(field("chart_kind", QFieldType.STRING).withPossibleValueSourceName("data_analysis_chart_kind"))
.withField(field("owner_user_id", QFieldType.STRING).withIsRequired(true))
.withField(field("shared", QFieldType.BOOLEAN).withDefaultValue(false))
.withField(field("read_owner_user_id", QFieldType.STRING).withIsEditable(false).withIsHidden(true))
.withAuditRules(new QAuditRules().withAuditLevel(AuditLevel.FIELD))
.withRecordSecurityLock(ownerLock("owner_user_id")));
.withRecordSecurityLock(ownerLock("owner_user_id").withLockScope(RecordSecurityLock.LockScope.WRITE))
.withRecordSecurityLock(ownerLock("read_owner_user_id").withLockScope(RecordSecurityLock.LockScope.READ)
.withNullValueBehavior(RecordSecurityLock.NullValueBehavior.ALLOW))
.withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_INSERT_RECORD,
new QCodeReference(com.kingsrook.qbits.dataanalysis.saved.SavedCustomizer.class))
.withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_UPDATE_RECORD,
new QCodeReference(com.kingsrook.qbits.dataanalysis.saved.SavedCustomizer.class)));
tables.add(base(config, SAVED_STAR, "Stars", "star", "data_analysis.saved")
.withRecordLabelFormatAndFields("%s", "user_id")
.withField(field("saved_id", QFieldType.INTEGER).withPossibleValueSourceName("data_analysis_saved").withIsRequired(true))
@@ -259,9 +266,13 @@ public final class DataAnalysisTables
.withField(field("recipients", QFieldType.TEXT))
.withField(field("enabled", QFieldType.BOOLEAN).withDefaultValue(false))
.withField(field("last_run_at", QFieldType.DATE_TIME))
.withField(field("last_status", QFieldType.STRING).withPossibleValueSourceName("data_analysis_run_status"))
.withField(field("last_status", QFieldType.TEXT).withIsEditable(false))
.withField(field("next_run_at", QFieldType.DATE_TIME))
.withRecordSecurityLock(ownerLock(SAVED + ".owner_user_id").withJoinNameChain(List.of(joinName(SCHEDULE)))));
.withRecordSecurityLock(ownerLock(SAVED + ".owner_user_id").withJoinNameChain(List.of(joinName(SCHEDULE))))
.withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_INSERT_RECORD,
new QCodeReference(com.kingsrook.qbits.dataanalysis.schedule.ScheduleCustomizer.class))
.withCustomizer(com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizers.PRE_UPDATE_RECORD,
new QCodeReference(com.kingsrook.qbits.dataanalysis.schedule.ScheduleCustomizer.class)));
tables.add(base(config, WATCH, "Watches", "notifications_active", "data_analysis.watch")
.withRecordLabelFormatAndFields("%s", "detector")
.withField(field("saved_id", QFieldType.INTEGER).withPossibleValueSourceName("data_analysis_saved")
@@ -305,7 +316,7 @@ public final class DataAnalysisTables
for(QTableMetaData table : tables)
{
List<String> fields = new ArrayList<>(table.getFields().keySet());
fields.removeAll(List.of("id", "createDate", "modifyDate"));
fields.removeAll(List.of("id", "createDate", "modifyDate", "read_owner_user_id"));
int headlineSize = Math.min(4, fields.size());
table.withSection(SectionFactory.defaultT1(fields.subList(0, headlineSize).toArray(String[]::new)));
if(fields.size() > headlineSize)
@@ -0,0 +1,144 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.processes;
import java.time.Instant;
import java.util.ArrayList;
import com.kingsrook.qbits.dataanalysis.DataAnalysisRuntime;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
import com.kingsrook.qqq.backend.core.actions.processes.BackendStep;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLine;
import com.kingsrook.qqq.backend.core.model.actions.processes.ProcessSummaryLineInterface;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.Status;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractLoadStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.AbstractTransformStep;
import com.kingsrook.qqq.backend.core.processes.implementations.etl.streamedwithfrontend.ProcessSummaryProviderInterface;
/*******************************************************************************
** Standard streamed ETL Review and Result actions for scheduled analyses.
*******************************************************************************/
public final class ScheduleActions
{
public static final String SEND = "data_analysis_schedule_send";
public static final String SCHEDULED = "data_analysis_run_schedules";
/*******************************************************************************
** Steps only.
*******************************************************************************/
private ScheduleActions()
{
}
/*******************************************************************************
** Review has no delivery side effects.
*******************************************************************************/
public static class Review extends AbstractTransformStep implements ProcessSummaryProviderInterface
{
private final ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
/*******************************************************************************
** Core extraction enforces record locks; verify the edit grant as well.
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
if(!PermissionsHelper.hasTablePermission(new UpdateInput(DataAnalysisTables.SCHEDULE), DataAnalysisTables.SCHEDULE, TablePermissionSubType.EDIT))
{
throw new QPermissionDeniedException("Schedule editing is unavailable.");
}
for(QRecord record : input.getRecords())
{
lines.add(new ProcessSummaryLine(Status.INFO, null, "Schedule " + record.getValueString("id") + " will be sent to authorized recipients."));
output.addRecord(record);
}
}
/*******************************************************************************
** Standard Review summary.
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean result)
{
return (result ? new ArrayList<>() : lines);
}
}
/*******************************************************************************
** Every selected record contributes its actual delivery status to Result.
*******************************************************************************/
public static class Result extends AbstractLoadStep implements ProcessSummaryProviderInterface
{
private final ArrayList<ProcessSummaryLineInterface> lines = new ArrayList<>();
/*******************************************************************************
** Reauthorize when executing after Review.
*******************************************************************************/
@Override
public void runOnePage(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
for(QRecord record : input.getRecords())
{
try
{
String status = DataAnalysisRuntime.schedules().sendNow(record.getValue("id"), Instant.now());
lines.add(new ProcessSummaryLine(status.startsWith("Failed") ? Status.ERROR : Status.OK, null, status));
output.addRecord(record);
}
catch(QException e)
{
lines.add(new ProcessSummaryLine(Status.ERROR, null, e.getMessage()));
}
}
}
/*******************************************************************************
** Actual results across streamed pages.
*******************************************************************************/
@Override
public ArrayList<ProcessSummaryLineInterface> getProcessSummary(RunBackendStepOutput output, boolean result)
{
return (lines);
}
}
/*******************************************************************************
** Hidden scheduler every five minutes.
*******************************************************************************/
public static class Scheduled implements BackendStep
{
/*******************************************************************************
** Count actual due schedule attempts.
*******************************************************************************/
@Override
public void run(RunBackendStepInput input, RunBackendStepOutput output) throws QException
{
output.addValue("count", DataAnalysisRuntime.schedules().runDue(Instant.now()));
}
}
}
@@ -0,0 +1,304 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.saved;
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qbits.dataanalysis.result.Metric;
import com.kingsrook.qbits.dataanalysis.result.PivotHeader;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ExportInput;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportDestination;
import com.kingsrook.qqq.backend.core.model.actions.reporting.ReportFormat;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData;
import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportField;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.QReportView;
import com.kingsrook.qqq.backend.core.model.metadata.reporting.ReportType;
/*******************************************************************************
** Export already-authorized typed results using core CSV and XLSX streamers.
*******************************************************************************/
public final class AnalysisExport
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private AnalysisExport()
{
}
/*******************************************************************************
** A result table, also reused by scheduled HTML summaries.
*******************************************************************************/
public record ResultTable(List<String> columns, List<List<String>> rows)
{
}
/*******************************************************************************
** Download bytes and metadata; filenames are fixed rather than user supplied.
*******************************************************************************/
public record File(byte[] bytes, String mimeType, String filename)
{
}
/*******************************************************************************
** Accept only the two documented export formats.
*******************************************************************************/
public static ReportFormat format(String value) throws QBadRequestException
{
if("csv".equalsIgnoreCase(value))
{
return (ReportFormat.CSV);
}
if("xlsx".equalsIgnoreCase(value))
{
return (ReportFormat.XLSX);
}
throw new QBadRequestException("format must be csv or xlsx.");
}
/*******************************************************************************
** Core's report streamers accept computed records without a source-table query.
*******************************************************************************/
public static File file(SavedResult result, String format) throws QException
{
ReportFormat type = format(format);
ResultTable table = table(result);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
var destination = new ReportDestination().withReportFormat(type).withReportOutputStream(bytes);
var view = new QReportView().withName("analysis").withLabel("Analysis").withType(ReportType.TABLE);
List<QFieldMetaData> fields = new ArrayList<>();
for(int i = 0; i < table.columns().size(); i++)
{
fields.add(new QFieldMetaData("c" + i, QFieldType.STRING).withLabel(table.columns().get(i)));
view.withColumn(new QReportField("c" + i).withLabel(table.columns().get(i)).withType(QFieldType.STRING));
}
var streamer = type.newReportStreamer();
streamer.preRun(destination, List.of(view));
streamer.start(new ExportInput().withReportDestination(destination), fields, "Analysis", view);
List<QRecord> records = new ArrayList<>();
for(List<String> row : table.rows())
{
QRecord record = new QRecord();
for(int i = 0; i < row.size(); i++)
{
record.withValue("c" + i, row.get(i));
}
records.add(record);
}
for(String warning : result.warnings())
{
records.add(new QRecord().withValue("c0", "Warning: " + warning));
}
streamer.addRecords(records);
streamer.finish();
return (new File(bytes.toByteArray(), type.getMimeType(), "analysis." + type.getExtension()));
}
/*******************************************************************************
** Preserve formatted values and suppressed cells; never export underlying raw data.
*******************************************************************************/
public static ResultTable table(SavedResult result)
{
List<String> columns = new ArrayList<>(List.of("Section", "Metric", "Value"));
List<List<String>> rows = new ArrayList<>();
if(result.columnProfile() != null)
{
var p = result.columnProfile();
metrics(rows, "Counts", p.getCounts());
if(p.getNumeric() != null)
{
metrics(rows, "Numeric", p.getNumeric().getMetrics());
}
if(p.getText() != null)
{
metrics(rows, "Text", p.getText().getMetrics());
}
if(p.getDates() != null)
{
metrics(rows, "Dates", p.getDates().getMetrics());
}
if(p.getBooleans() != null)
{
metrics(rows, "Boolean", p.getBooleans());
}
p.getTopValues().forEach(v -> rows.add(row("Top values", v.getLabel(), v.getCount())));
}
else if(result.tableProfile() != null)
{
var p = result.tableProfile();
metrics(rows, "Table", List.of(p.getRowCount()));
for(var field : p.getFields())
{
rows.add(row(field.getLabel(), "Completeness", field.getCompleteness()));
rows.add(row(field.getLabel(), "Distinct", field.getDistinct()));
rows.add(row(field.getLabel(), "Uniqueness", field.getUniqueness()));
}
}
else if(result.pivot() != null)
{
var p = result.pivot();
columns = new ArrayList<>(List.of("Row"));
for(var headers : p.getColumnHeaders())
{
for(var measure : p.getMeasures())
{
columns.add(label(headers) + " / " + measure.getLabel());
}
}
for(int r = 0; r < p.getRowHeaders().size(); r++)
{
List<String> row = new ArrayList<>();
row.add(label(p.getRowHeaders().get(r)));
for(int c = 0; c < p.getColumnHeaders().size(); c++)
{
final int ri = r;
final int ci = c;
var cell = p.getCells().stream().filter(x -> x.getRowIndex() == ri && x.getColumnIndex() == ci).findFirst();
for(int m = 0; m < p.getMeasures().size(); m++)
{
row.add(cell.isPresent() ? value(cell.get().getValues().get(m)) : "");
}
}
rows.add(row);
}
}
else if(result.compare() != null)
{
columns = List.of("Field", "Value", "Metric", "A", "B", "Delta", "Percent delta");
for(var field : result.compare().getFields())
{
for(var comparison : field.getValues())
{
for(var metric : comparison.getSideA().getMetrics())
{
var b = comparison.getSideB().getMetrics().stream().filter(m -> Objects.equals(m.getKey(), metric.getKey())).findFirst().orElse(null);
rows.add(row(field.getField(), comparison.getLabel(), metric.getLabel(), value(metric), value(b), value(comparison.getDelta()), value(comparison.getPercentDelta())));
}
}
}
}
else if(result.correlation() != null)
{
columns = List.of("X", "Y", "Paired rows", "Correlation");
result.correlation().getCells().forEach(c -> rows.add(row(c.getXField(), c.getYField(), c.getN(), c.getR())));
}
else if(result.crosstab() != null)
{
columns = List.of("Row", "Column", "Count", "Row share", "Column share");
var p = result.crosstab();
p.getCells().forEach(c -> rows.add(row(p.getRows().get(c.getRowIndex()).getLabel(), p.getColumns().get(c.getColumnIndex()).getLabel(),
value(c.getCount()), value(c.getRowShare()), value(c.getColumnShare()))));
}
else if(result.shift() != null)
{
for(var bucket : result.shift().getBuckets())
{
metrics(rows, bucket.getLabel(), bucket.getMetrics());
}
if(result.shift().getPsi() != null)
{
metrics(rows, "Shift", List.of(result.shift().getPsi()));
}
}
return (new ResultTable(columns, rows));
}
/*******************************************************************************
** Headline metrics shared by standard chart widgets and email summaries.
*******************************************************************************/
public static List<Metric> headlines(SavedResult result)
{
List<Metric> metrics = new ArrayList<>();
if(result.columnProfile() != null)
{
metrics.addAll(result.columnProfile().getCounts());
if(result.columnProfile().getNumeric() != null)
{
metrics.addAll(result.columnProfile().getNumeric().getMetrics().stream()
.filter(m -> List.of("mean", "median", "min", "max").contains(m.getKey())).toList());
}
}
else if(result.tableProfile() != null)
{
metrics.add(result.tableProfile().getRowCount());
}
else if(result.pivot() != null)
{
metrics.addAll(result.pivot().getGrandTotals());
}
else if(result.compare() != null)
{
result.compare().getFields().stream().limit(4).forEach(f -> f.getValues().stream().limit(1).forEach(v -> metrics.add(v.getDelta())));
}
else if(result.shift() != null)
{
metrics.add(result.shift().getPsi());
}
else
{
int count = result.correlation() != null ? result.correlation().getFields().size() : result.crosstab() != null ? result.crosstab().getCells().size() : 0;
if(result.correlation() != null || result.crosstab() != null)
{
metrics.add(new Metric().withKey("visible").withLabel(result.correlation() != null ? "Numeric fields" : "Visible combinations")
.withValue(count).withValueType(com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withFormatted(String.valueOf(count)));
}
}
return (metrics.stream().filter(Objects::nonNull).toList());
}
/*******************************************************************************
** No nulls in report cells; unknown and suppressed are distinct from zero.
*******************************************************************************/
private static List<String> row(Object... values)
{
return (java.util.Arrays.stream(values).map(v -> v == null ? "" : String.valueOf(v)).toList());
}
/*******************************************************************************
** Render a tuple using the engine's already-authorized display labels.
*******************************************************************************/
private static String label(List<PivotHeader> headers)
{
return (headers.stream().map(PivotHeader::getLabel).collect(java.util.stream.Collectors.joining(" / ")));
}
/*******************************************************************************
** Metrics always use their semantic formatted value.
*******************************************************************************/
public static String value(Metric metric)
{
return (metric == null ? "" : Boolean.TRUE.equals(metric.getSuppressed()) ? "Suppressed" : Objects.toString(metric.getFormatted(), ""));
}
/*******************************************************************************
** Profile metrics become one row per headline statistic.
*******************************************************************************/
private static void metrics(List<List<String>> rows, String section, List<Metric> metrics)
{
metrics.forEach(m -> rows.add(row(section, m.getLabel(), value(m))));
}
}
@@ -0,0 +1,151 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.saved;
import java.io.Serializable;
import java.time.Instant;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.kingsrook.qbits.dataanalysis.DataAnalysisService;
import com.kingsrook.qbits.dataanalysis.engine.DrillFilters;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.watch.WatchRecords;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
import com.kingsrook.qqq.backend.core.actions.tables.DeleteAction;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.QueryAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
/*******************************************************************************
** Saved definitions always execute in the calling context, including shared reads.
*******************************************************************************/
public class SavedAnalyses
{
private final DataAnalysisService service;
/*******************************************************************************
** Use the instance's existing engine and cache policy.
*******************************************************************************/
public SavedAnalyses(DataAnalysisService service)
{
this.service = service;
}
/*******************************************************************************
** Standard core READ and record locks precede source-table authorization.
*******************************************************************************/
public SavedResult run(Serializable id) throws QException
{
return (SavedDefinition.parse(WatchRecords.get(DataAnalysisTables.SAVED, id)).run(service, false));
}
/*******************************************************************************
** Readable list rows contain presentation metadata, never cached owner results.
*******************************************************************************/
public record Entry(Serializable id, String name, String kind, String tableName, String tableLabel, String owner,
Boolean shared, Boolean starred, Instant updated)
{
}
/*******************************************************************************
** Only stars belonging to this viewer decorate their core-visible saved rows.
*******************************************************************************/
public List<Entry> list() throws QException
{
Set<Serializable> stars = new HashSet<>();
for(QRecord row : query(DataAnalysisTables.SAVED_STAR, DrillFilters.condition("user_id", QCriteriaOperator.EQUALS, user())))
{
stars.add(row.getValue("saved_id"));
}
List<Entry> result = new ArrayList<>();
for(QRecord row : query(DataAnalysisTables.SAVED, new QQueryFilter().withOrderBy(new QFilterOrderBy("modifyDate", false))))
{
String tableName = row.getValueString("table_name");
var table = QContext.getQInstance().getTable(tableName);
result.add(new Entry(row.getValue("id"), row.getValueString("name"), row.getValueString("kind"), tableName,
table == null ? tableName : table.getLabel(), row.getValueString("owner_user_id"), row.getValueBoolean("shared"),
stars.contains(row.getValue("id")), row.getValueInstant("modifyDate")));
}
return (result);
}
/*******************************************************************************
** Idempotent per-runtime PUT/DELETE, with standard insert/delete permissions.
*******************************************************************************/
public synchronized Boolean star(Serializable id, boolean starred) throws QException
{
WatchRecords.get(DataAnalysisTables.SAVED, id);
QQueryFilter filter = DrillFilters.and(DrillFilters.condition("saved_id", QCriteriaOperator.EQUALS, id),
DrillFilters.condition("user_id", QCriteriaOperator.EQUALS, user()));
List<QRecord> existing = query(DataAnalysisTables.SAVED_STAR, filter);
if(starred && existing.isEmpty())
{
InsertInput input = new InsertInput(DataAnalysisTables.SAVED_STAR).withRecords(List.of(new QRecord().withValue("saved_id", id).withValue("user_id", user())));
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
QRecord record = new InsertAction().execute(input).getRecords().getFirst();
if(record.getErrors() != null && !record.getErrors().isEmpty())
{
throw new QException("Could not save the star.");
}
}
else if(!starred && !existing.isEmpty())
{
DeleteInput input = new DeleteInput(DataAnalysisTables.SAVED_STAR).withPrimaryKeys(existing.stream().map(r -> r.getValue("id")).toList());
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
new DeleteAction().execute(input);
}
return (starred);
}
/*******************************************************************************
** Shared enforced query helper for visible records, not a security-filter layer.
*******************************************************************************/
public static List<QRecord> query(String table, QQueryFilter filter) throws QException
{
QueryInput input = new QueryInput(table).withShouldFetchHeavyFields(true).withFilter(filter);
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
return (new QueryAction().execute(input).getRecords());
}
/*******************************************************************************
** An authenticated identity is required even in an unprotected host app.
*******************************************************************************/
public static String user() throws QPermissionDeniedException
{
var session = QContext.getQSession();
if(session == null || session.getUser() == null || session.getUser().getIdReference() == null)
{
throw new QPermissionDeniedException("Sign in to use saved analyses.");
}
return (session.getUser().getIdReference());
}
}
@@ -0,0 +1,91 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.saved;
import java.util.List;
import java.util.Optional;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions;
import com.kingsrook.qbits.dataanalysis.watch.WatchRecords;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
/*******************************************************************************
** Validate all core CRUD and maintain the read-lock projection atomically.
*******************************************************************************/
public class SavedCustomizer implements TableCustomizerInterface
{
/*******************************************************************************
** Inserts cannot forge an owner or the derived shared-read key.
*******************************************************************************/
@Override
public List<QRecord> preInsert(InsertInput input, List<QRecord> records, boolean preview) throws QException
{
for(QRecord row : records)
{
validate(row, row);
}
return (records);
}
/*******************************************************************************
** Merge partial edits so changing shared or owner also changes the read lock.
*******************************************************************************/
@Override
public List<QRecord> preUpdate(UpdateInput input, List<QRecord> records, boolean preview, Optional<List<QRecord>> oldRecords) throws QException
{
for(QRecord patch : records)
{
QRecord merged = new QRecord(WatchRecords.get(DataAnalysisTables.SAVED, patch.getValue("id")));
merged.getValues().putAll(patch.getValues());
validate(patch, merged);
}
return (records);
}
/*******************************************************************************
** Report a plain validation message on the standard record editor.
*******************************************************************************/
private void validate(QRecord output, QRecord merged)
{
try
{
String owner = merged.getValueString("owner_user_id");
var session = QContext.getQSession();
if(owner == null || owner.isBlank() || session == null || (!(session.getSecurityKeyValues() != null && session.getSecurityKeyValues().getOrDefault(DataAnalysisPermissions.ADMIN, List.of()).contains(true))
&& (session.getUser() == null || !owner.equals(session.getUser().getIdReference()))))
{
throw new QException("Only an administrator may assign another analysis owner.");
}
SavedDefinition.parse(merged);
output.withValue("read_owner_user_id", Boolean.TRUE.equals(merged.getValueBoolean("shared")) ? null : owner);
}
catch(QException e)
{
output.addError(new BadInputStatusMessage(e.getMessage()));
}
}
}
@@ -0,0 +1,313 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.saved;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import com.kingsrook.qbits.dataanalysis.DataAnalysisService;
import com.kingsrook.qbits.dataanalysis.result.AnalysisRequest;
import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest;
import com.kingsrook.qbits.dataanalysis.result.CompareRequest;
import com.kingsrook.qbits.dataanalysis.result.CorrelationRequest;
import com.kingsrook.qbits.dataanalysis.result.CrosstabRequest;
import com.kingsrook.qbits.dataanalysis.result.PivotRequest;
import com.kingsrook.qbits.dataanalysis.result.ShiftRequest;
import com.kingsrook.qbits.dataanalysis.result.TableProfileRequest;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
** One parser for persisted definitions, exports, schedules and SQL requests.
*******************************************************************************/
public record SavedDefinition(String kind, AnalysisRequest request)
{
private static final Map<String, Class<? extends AnalysisRequest>> TYPES = Map.of(
"COLUMN_PROFILE", ColumnProfileRequest.class, "TABLE_PROFILE", TableProfileRequest.class,
"PIVOT", PivotRequest.class, "COMPARE", CompareRequest.class, "CORRELATION", CorrelationRequest.class,
"CROSSTAB", CrosstabRequest.class, "SHIFT", ShiftRequest.class);
private static final Set<String> FIELD_KEYS = Set.of("fieldName", "field", "dateField", "rowField", "columnField");
public static final String FIELD_WARNING = "Some fields are not available to you";
/*******************************************************************************
** The definition owns the typed filter; legacy filter_json remains supported.
*******************************************************************************/
public static SavedDefinition parse(QRecord row) throws QException
{
Map<String, Object> body = object(row.getValueString("definition_json"));
if(body.get("filter") == null && row.getValueString("filter_json") != null)
{
body.put("filter", object(row.getValueString("filter_json")));
}
return (parse(row.getValueString("kind"), row.getValueString("table_name"), body));
}
/*******************************************************************************
** Reject malformed JSON with a plain, stable message.
*******************************************************************************/
@SuppressWarnings("unchecked")
public static Map<String, Object> object(String json) throws QBadRequestException
{
try
{
Object value = JsonUtils.toObject(json, Object.class);
if(value instanceof Map<?, ?> map)
{
return (new LinkedHashMap<>((Map<String, Object>) map));
}
}
catch(Exception expected)
{
// Normalize parser diagnostics; never return raw saved JSON in errors.
}
throw new QBadRequestException("definition_json must be a JSON object.");
}
/*******************************************************************************
** Validate metadata existence before saving, without querying source records.
*******************************************************************************/
public static SavedDefinition parse(String kind, String table, Map<String, Object> body) throws QException
{
if(!TYPES.containsKey(kind == null ? "" : kind))
{
throw new QBadRequestException("Choose a supported analysis kind.");
}
if(table == null || QContext.getQInstance().getTable(table) == null)
{
throw new QBadRequestException("Choose an existing analysis table.");
}
if(body.get("tableName") != null && !table.equals(body.get("tableName")))
{
throw new QBadRequestException("The definition table must match table_name.");
}
Map<String, Object> copy = new LinkedHashMap<>(body);
copy.put("tableName", table);
try
{
validateJson(copy);
AnalysisRequest request = JsonUtils.toObject(JsonUtils.toJson(copy), TYPES.get(kind));
requireShape(request);
for(String field : fields(copy))
{
String[] parts = field.split("\\.", 2);
var metadata = QContext.getQInstance().getTable(parts.length == 2 ? parts[0] : table);
if(metadata == null || metadata.getField(parts.length == 2 ? parts[1] : field) == null)
{
throw new IllegalArgumentException("A definition field does not exist.");
}
}
return (new SavedDefinition(kind, request));
}
catch(Exception e)
{
throw new QBadRequestException("Invalid analysis definition: " + (e instanceof IllegalArgumentException ? e.getMessage() : "check the request fields and types."));
}
}
/*******************************************************************************
** Reject coercible scalar types and malformed predicates before Jackson conversion.
*******************************************************************************/
private static void validateJson(Object value)
{
if(value instanceof Map<?, ?> map)
{
for(var entry : map.entrySet())
{
String name = String.valueOf(entry.getKey());
Object item = entry.getValue();
if(item == null)
{
continue;
}
if((FIELD_KEYS.contains(name) || Set.of("tableName", "bucket", "operator").contains(name)) && !(item instanceof String))
{
throw new IllegalArgumentException("Field names and operators must be strings.");
}
if(name.equals("fields") && (!(item instanceof List<?> list) || list.stream().anyMatch(v -> !(v instanceof String))))
{
throw new IllegalArgumentException("fields must be an array of field names.");
}
if(Set.of("filter", "filterA", "filterB").contains(name) && !(item instanceof Map<?, ?>))
{
throw new IllegalArgumentException("Filters must be query filter objects.");
}
if(name.equals("criteria") && (!(item instanceof List<?> list) || list.stream().anyMatch(v -> !(v instanceof Map<?, ?> c)
|| !(c.get("fieldName") instanceof String) || !(c.get("operator") instanceof String))))
{
throw new IllegalArgumentException("Filter criteria require a field and operator.");
}
if(Set.of("exact", "showTotals").contains(name) && !(item instanceof Boolean))
{
throw new IllegalArgumentException("Flags must be booleans.");
}
validateJson(item);
}
}
else if(value instanceof List<?> list)
{
list.forEach(SavedDefinition::validateJson);
}
}
/*******************************************************************************
** Required selections must not turn an invalid draft into a different query.
*******************************************************************************/
private static void requireShape(AnalysisRequest request)
{
boolean valid = switch(request)
{
case ColumnProfileRequest r -> r.getFieldName() != null;
case TableProfileRequest r -> r.getFields() == null || !r.getFields().isEmpty();
case CompareRequest r -> r.getFields() != null && !r.getFields().isEmpty();
case CorrelationRequest r -> r.getFields() != null && r.getFields().size() >= 2;
case CrosstabRequest r -> r.getRowField() != null && r.getColumnField() != null;
case ShiftRequest r -> r.getField() != null && r.getDateField() != null;
case PivotRequest r -> r.getValues() != null && !r.getValues().isEmpty()
&& r.getValues().stream().allMatch(v -> v.getOperator() != null && (v.getOperator().name().equals("COUNT") || v.getField() != null));
default -> false;
};
if(!valid)
{
throw new IllegalArgumentException("Choose the fields and measures required by this analysis kind.");
}
}
/*******************************************************************************
** Include selected and filter fields, including both compare populations.
*******************************************************************************/
public static Set<String> fields(Object value)
{
Set<String> names = new LinkedHashSet<>();
if(value instanceof Map<?, ?> map)
{
for(var entry : map.entrySet())
{
if(FIELD_KEYS.contains(entry.getKey()) && entry.getValue() instanceof String name)
{
names.add(name);
}
else if("fields".equals(entry.getKey()) && entry.getValue() instanceof List<?> list)
{
list.forEach(name -> names.add((String) name));
}
else
{
names.addAll(fields(entry.getValue()));
}
}
}
else if(value instanceof List<?> list)
{
list.forEach(item -> names.addAll(fields(item)));
}
return (names);
}
/*******************************************************************************
** A fresh tree makes field reduction request-local and cannot change storage.
*******************************************************************************/
public Map<String, Object> body() throws QException
{
return (object(JsonUtils.toJson(request)));
}
/*******************************************************************************
** Filter rights are not weakened: an unavailable predicate prevents execution.
** Required denied axes return an empty typed envelope rather than change meaning.
*******************************************************************************/
@SuppressWarnings("unchecked")
public SavedResult run(DataAnalysisService service, boolean strict) throws QException
{
service.capabilities(request.getTableName());
Map<String, Object> body = body();
Set<String> denied = new LinkedHashSet<>();
for(String field : fields(body))
{
if(!service.isAvailableField(request.getTableName(), field))
{
denied.add(field);
}
}
if(strict && !denied.isEmpty())
{
throw new QBadRequestException(FIELD_WARNING);
}
List<String> warnings = denied.isEmpty() ? List.of() : List.of(FIELD_WARNING);
for(String filter : List.of("filter", "filterA", "filterB"))
{
if(fields(body.get(filter)).stream().anyMatch(denied::contains))
{
return (SavedResult.of(kind, null, warnings));
}
}
if(body.get("fields") instanceof List<?> list)
{
body.put("fields", list.stream().filter(f -> !denied.contains(f)).toList());
}
for(String axis : List.of("rows", "columns", "values"))
{
if(body.get(axis) instanceof List<?> list)
{
body.put(axis, list.stream().filter(item -> fields(item).stream().noneMatch(denied::contains)).toList());
}
}
if(FIELD_KEYS.stream().anyMatch(key -> denied.contains(body.get(key))))
{
return (SavedResult.of(kind, null, warnings));
}
AnalysisRequest reduced;
try
{
reduced = JsonUtils.toObject(JsonUtils.toJson(body), TYPES.get(kind));
requireShape(reduced);
}
catch(Exception e)
{
return (SavedResult.of(kind, null, warnings));
}
Object result = switch(reduced)
{
case ColumnProfileRequest r ->
{
var backend = service.backend(r.getTableName());
yield service.compute(r, backend, backend.scope(r.getTableName(), r.getFilter(), Boolean.TRUE.equals(r.getExact())));
}
case TableProfileRequest r ->
{
var backend = service.backend(r.getTableName());
yield service.compute(r, backend, backend.scope(r.getTableName(), r.getFilter(), Boolean.TRUE.equals(r.getExact())));
}
case PivotRequest r -> service.pivot(r);
case CompareRequest r -> service.compare(r);
case CorrelationRequest r -> service.correlation(r);
case CrosstabRequest r -> service.crosstab(r);
case ShiftRequest r -> service.shift(r);
default -> throw new QBadRequestException("Unsupported analysis kind.");
};
return (SavedResult.of(kind, result, warnings));
}
}
@@ -0,0 +1,60 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.saved;
import java.util.List;
import com.kingsrook.qbits.dataanalysis.result.ColumnProfile;
import com.kingsrook.qbits.dataanalysis.result.CompareResult;
import com.kingsrook.qbits.dataanalysis.result.CorrelationResult;
import com.kingsrook.qbits.dataanalysis.result.CrosstabResult;
import com.kingsrook.qbits.dataanalysis.result.PivotResult;
import com.kingsrook.qbits.dataanalysis.result.ShiftResult;
import com.kingsrook.qbits.dataanalysis.result.TableProfile;
/*******************************************************************************
** Discriminated result: exactly the member named by kind is populated when runnable.
*******************************************************************************/
public record SavedResult(String kind, List<String> warnings, ColumnProfile columnProfile, TableProfile tableProfile,
PivotResult pivot, CompareResult compare, CorrelationResult correlation, CrosstabResult crosstab, ShiftResult shift)
{
/*******************************************************************************
** Preserve typed engine results without an untyped JSON payload.
*******************************************************************************/
public static SavedResult of(String kind, Object result, List<String> warnings)
{
java.util.ArrayList<String> notices = new java.util.ArrayList<>(warnings);
if(result instanceof ColumnProfile p)
{
notices.addAll(p.getWarnings());
}
else if(result instanceof TableProfile p)
{
notices.addAll(p.getWarnings());
}
else if(result instanceof PivotResult p)
{
notices.addAll(p.getWarnings());
}
return (new SavedResult(kind, notices.stream().distinct().toList(), result instanceof ColumnProfile p ? p : null, result instanceof TableProfile p ? p : null,
result instanceof PivotResult p ? p : null, result instanceof CompareResult p ? p : null,
result instanceof CorrelationResult p ? p : null, result instanceof CrosstabResult p ? p : null, result instanceof ShiftResult p ? p : null));
}
}
@@ -0,0 +1,147 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.saved;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
import com.kingsrook.qbits.dataanalysis.DataAnalysisService;
import com.kingsrook.qqq.backend.core.actions.interfaces.AggregateInterface;
import com.kingsrook.qqq.backend.core.actions.tables.AggregateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.aggregate.AggregateOutput;
/*******************************************************************************
** Request-local capture of core's actual aggregate SQL, never a separate SQL planner.
*******************************************************************************/
public final class SqlHandoff
{
private static final ThreadLocal<List<Statement>> CAPTURE = new ThreadLocal<>();
private static final String NOTE = "Record security filters are applied by QQQ at run time. This SQL is for reading; ? placeholders remain bound by QQQ.";
/*******************************************************************************
** Utility class.
*******************************************************************************/
private SqlHandoff()
{
}
/*******************************************************************************
** Statement ranking prefers the main grouped matrix, then the widest metric batch.
*******************************************************************************/
private record Statement(String table, String sql, int dimensions, int measures)
{
}
/*******************************************************************************
** Transported capability, statement and security note.
*******************************************************************************/
public record Result(Boolean available, String sql, String note)
{
}
/*******************************************************************************
** Installation plus its configured access permission are both required.
*******************************************************************************/
public static boolean installedAndAllowed()
{
var instance = QContext.getQInstance();
var app = instance.getApp("sqlWorkbench");
boolean installed = app != null || instance.getTables().keySet().stream().anyMatch(t -> t.startsWith("sql_workbench_"));
String base = app != null && app.getPermissionRules() != null && app.getPermissionRules().getPermissionBaseName() != null
? app.getPermissionRules().getPermissionBaseName() : "sql_workbench";
return (installed && QContext.getQSession() != null && QContext.getQSession().hasPermission(base + ".hasAccess"));
}
/*******************************************************************************
** The backend metadata inheritance check avoids a production RDBMS dependency.
*******************************************************************************/
public static Result execute(SavedDefinition definition, DataAnalysisService service) throws QException
{
if(!installedAndAllowed())
{
return (new Result(false, null, "SQL Workbench is not installed or available to you."));
}
service.capabilities(definition.request().getTableName());
var table = QContext.getQInstance().getTable(definition.request().getTableName());
Class<?> type = QContext.getQInstance().getBackend(table.getBackendName()).getClass();
boolean rdbms = false;
while(type != null)
{
rdbms |= type.getSimpleName().equals("RDBMSBackendMetaData");
type = type.getSuperclass();
}
if(!rdbms)
{
return (new Result(false, null, "SQL is unavailable for a non-RDBMS backend."));
}
List<Statement> statements = new ArrayList<>();
CAPTURE.set(statements);
try
{
definition.run(service, true);
}
finally
{
CAPTURE.remove();
}
java.util.Comparator<Statement> ranking = List.of("PIVOT", "CROSSTAB", "SHIFT").contains(definition.kind())
? java.util.Comparator.comparingInt(Statement::dimensions).thenComparingInt(Statement::measures)
: java.util.Comparator.comparingInt(Statement::measures).thenComparingInt(s -> -s.dimensions());
Statement main = statements.stream().filter(s -> s.table().equals(table.getName())).max(ranking).orElse(null);
return (main == null ? new Result(false, null, "Core did not expose SQL for this aggregate. " + NOTE) : new Result(true, main.sql(), NOTE));
}
/*******************************************************************************
** Execute the ordinary authorized core action, then read its backend QueryStat.
** Core exposes QueryStat on the backend but not its action reference; this narrow
** bridge never changes action state, global logging, or QueryStatManager settings.
*******************************************************************************/
public static AggregateOutput aggregate(AggregateInput input) throws QException
{
if(CAPTURE.get() != null)
{
input.setTableMetaData(input.getTable().clone().withCapability(com.kingsrook.qqq.backend.core.model.metadata.tables.Capability.QUERY_STATS));
}
AggregateAction action = new AggregateAction();
AggregateOutput output = action.execute(input);
if(CAPTURE.get() != null)
{
try
{
Field field = AggregateAction.class.getDeclaredField("aggregateInterface");
field.setAccessible(true);
var stat = ((AggregateInterface) field.get(action)).getQueryStat();
if(stat != null && stat.getQueryText() != null)
{
CAPTURE.get().add(new Statement(input.getTableName(), stat.getQueryText(), input.getGroupBys().size(), input.getAggregates().size()));
}
}
catch(ReflectiveOperationException | RuntimeException expected)
{
// A future core change makes handoff unavailable rather than invent SQL.
}
}
return (output);
}
}
@@ -0,0 +1,116 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.schedule;
import java.time.Instant;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition;
import com.kingsrook.qbits.dataanalysis.watch.WatchRecords;
import com.kingsrook.qqq.backend.core.actions.customizers.TableCustomizerInterface;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.statusmessages.BadInputStatusMessage;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
/*******************************************************************************
** Core CRUD validates recipients and computes the next occurrence on configuration edits.
*******************************************************************************/
public class ScheduleCustomizer implements TableCustomizerInterface
{
private static final Set<String> CONFIG = Set.of("saved_id", "cron", "frequency", "day", "time", "time_zone", "enabled", "recipients", "format");
/*******************************************************************************
** Disabled empty drafts may be completed later.
*******************************************************************************/
@Override
public List<QRecord> preInsert(InsertInput input, List<QRecord> records, boolean preview) throws QException
{
for(QRecord row : records)
{
validate(row, row);
}
return (records);
}
/*******************************************************************************
** Status-only scheduler writes do not move the next occurrence twice.
*******************************************************************************/
@Override
public List<QRecord> preUpdate(UpdateInput input, List<QRecord> records, boolean preview, Optional<List<QRecord>> oldRecords) throws QException
{
for(QRecord patch : records)
{
if(patch.getValues().keySet().stream().anyMatch(CONFIG::contains))
{
QRecord row = new QRecord(WatchRecords.get(DataAnalysisTables.SCHEDULE, patch.getValue("id")));
row.getValues().putAll(patch.getValues());
validate(patch, row);
}
}
return (records);
}
/*******************************************************************************
** User ids are stored as a JSON array, not an unchecked email recipient list.
*******************************************************************************/
public static List<String> recipients(QRecord row) throws QException
{
try
{
Object value = JsonUtils.toObject(row.getValueString("recipients"), Object.class);
if(value instanceof List<?> list && !list.isEmpty() && list.size() <= 100
&& list.stream().allMatch(v -> v instanceof String s && !s.isBlank()))
{
return (list.stream().map(String.class::cast).distinct().toList());
}
}
catch(Exception expected)
{
// Use the same plain error for JSON and shape errors.
}
throw new QException("Recipients must be a JSON array of 1 to 100 user ids.");
}
/*******************************************************************************
** Scheduling references a real, visible saved analysis.
*******************************************************************************/
private void validate(QRecord output, QRecord row)
{
try
{
SavedDefinition.parse(WatchRecords.get(DataAnalysisTables.SAVED, row.getValue("saved_id")));
if(Boolean.TRUE.equals(row.getValueBoolean("enabled")) || row.getValueString("frequency") != null || row.getValueString("cron") != null)
{
recipients(row);
output.withValue("next_run_at", ScheduleTiming.next(row, Instant.now()));
}
}
catch(QException e)
{
output.addError(new BadInputStatusMessage(e.getMessage()));
}
}
}
@@ -0,0 +1,305 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.schedule;
import java.io.Serializable;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig;
import com.kingsrook.qbits.dataanalysis.DataAnalysisService;
import com.kingsrook.qbits.dataanalysis.engine.DrillFilters;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.permissions.DataAnalysisPermissions;
import com.kingsrook.qbits.dataanalysis.saved.AnalysisExport;
import com.kingsrook.qbits.dataanalysis.saved.SavedAnalyses;
import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition;
import com.kingsrook.qbits.dataanalysis.saved.SavedResult;
import com.kingsrook.qbits.dataanalysis.watch.WatchRecords;
import com.kingsrook.qqq.backend.core.actions.messaging.SendMessageAction;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement;
import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper;
import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.alerts.AlertRecordRef;
import com.kingsrook.qqq.backend.core.alerts.AlertSeverity;
import com.kingsrook.qqq.backend.core.alerts.AlertSignal;
import com.kingsrook.qqq.backend.core.alerts.QAlerts;
import com.kingsrook.qqq.backend.core.context.CapturedContext;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.exceptions.QPermissionDeniedException;
import com.kingsrook.qqq.backend.core.model.actions.messaging.Attachment;
import com.kingsrook.qqq.backend.core.model.actions.messaging.Content;
import com.kingsrook.qqq.backend.core.model.actions.messaging.Party;
import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput;
import com.kingsrook.qqq.backend.core.model.actions.messaging.email.EmailContentRole;
import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.model.session.QUser;
import com.kingsrook.qqq.backend.core.modules.authentication.QAuthenticationModuleDispatcher;
/*******************************************************************************
** Scheduled work uses owner and recipient sessions; maintenance only inventories rows.
*******************************************************************************/
public class ScheduleRunner
{
public static final String ALERT_TYPE = "data.analysis.schedule.failed";
private final DataAnalysisQBitConfig config;
private final DataAnalysisService service;
/*******************************************************************************
** One instance serializes manual and automatic delivery on this application node.
*******************************************************************************/
public ScheduleRunner(DataAnalysisQBitConfig config, DataAnalysisService service)
{
this.config = config;
this.service = service;
}
/*******************************************************************************
** Enumerate enabled due rows with a maintenance session restricted to metadata.
*******************************************************************************/
public synchronized Integer runDue(Instant now) throws QException
{
QSession maintenance = new QSession().withUser(new QUser().withIdReference("data-analysis-scheduler"))
.withPermissions("data_analysis.saved.read", "data_analysis.saved.edit")
.withSecurityKeyValue(DataAnalysisPermissions.ADMIN, true);
if(QContext.getQSession().getBackendVariants() != null)
{
maintenance.setBackendVariants(new java.util.HashMap<>(QContext.getQSession().getBackendVariants()));
}
int[] count = {0};
QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), maintenance), () ->
{
var filter = DrillFilters.and(DrillFilters.condition("enabled", QCriteriaOperator.EQUALS, true),
DrillFilters.condition("next_run_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now));
for(QRecord row : SavedAnalyses.query(DataAnalysisTables.SCHEDULE, filter))
{
sendNow(row.getValue("id"), now);
count[0]++;
}
});
return (count[0]);
}
/*******************************************************************************
** Manual delivery is authorized before source work; all attempts record their outcome.
*******************************************************************************/
public synchronized String sendNow(Serializable id, Instant now) throws QException
{
UpdateInput authorization = new UpdateInput(DataAnalysisTables.SCHEDULE);
authorization.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
if(!PermissionsHelper.hasTablePermission(authorization, DataAnalysisTables.SCHEDULE, TablePermissionSubType.EDIT))
{
throw new QPermissionDeniedException("Schedule editing is unavailable.");
}
QRecord schedule = WatchRecords.get(DataAnalysisTables.SCHEDULE, id);
String status;
boolean failed = false;
try
{
status = deliver(schedule);
}
catch(Exception e)
{
failed = true;
status = "Failed: " + (e instanceof QException ? e.getMessage() : "Delivery unavailable; check the host configuration.");
}
Instant next = null;
try
{
next = ScheduleTiming.next(schedule, now);
}
catch(QException e)
{
failed = true;
status += "; " + e.getMessage();
}
QRecord patch = new QRecord().withValue("id", id).withValue("last_run_at", now).withValue("last_status", status).withValue("next_run_at", next);
UpdateInput input = new UpdateInput(DataAnalysisTables.SCHEDULE).withRecords(List.of(patch));
input.setPermissionEnforcement(PermissionEnforcement.ENFORCE);
QRecord stored = new UpdateAction().execute(input).getRecords().getFirst();
if(stored.getErrors() != null && !stored.getErrors().isEmpty())
{
throw new QException("Schedule status could not be stored.");
}
if(failed)
{
QAlerts.raise(new AlertSignal(ALERT_TYPE).withDedupKey("data_analysis_schedule:" + id).withSeverity(AlertSeverity.ERROR)
.withTitle("Analysis schedule failed").withSummary(status).withRecordRef(AlertRecordRef.of(DataAnalysisTables.SCHEDULE, id)));
}
return (status);
}
/*******************************************************************************
** The host resolves identities; an unavailable identity never becomes a system user.
*******************************************************************************/
private QSession session(String id) throws QException
{
QSession session;
try
{
session = config.getOwnerSessionSupplier() == null
? new QAuthenticationModuleDispatcher().getQModule(QContext.getQInstance().getAuthentication()).createAutomatedSessionForUser(QContext.getQInstance(), id)
: config.getOwnerSessionSupplier().apply(id);
}
catch(Exception e)
{
throw new QException("User session unavailable.");
}
if(session == null || session.getUser() == null || !Objects.equals(id, session.getUser().getIdReference())
|| session.getSecurityKeyValues() == null || !session.getSecurityKeyValues().getOrDefault(DataAnalysisTables.USER_KEY, List.of()).contains(id))
{
throw new QException("User session unavailable.");
}
return (session);
}
/*******************************************************************************
** Re-run for each recipient to respect their record locks, not just table grants.
*******************************************************************************/
private String deliver(QRecord schedule) throws QException
{
if(config.getMessagingProviderName() == null || QContext.getQInstance().getMessagingProvider(config.getMessagingProviderName()) == null)
{
throw new QException("No messaging provider configured.");
}
if(config.getFromAddress() == null || config.getRecipientAddressSupplier() == null)
{
throw new QException("Configure fromAddress and recipientAddressSupplier for scheduled email.");
}
if(List.of("CSV", "XLSX").contains(Objects.toString(schedule.getValueString("format"), "")))
{
var provider = new com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher()
.getMessagingProviderInterface(QContext.getQInstance().getMessagingProvider(config.getMessagingProviderName()));
if(provider.getClass().equals(com.kingsrook.qqq.backend.core.model.metadata.messaging.email.EmailMessagingProvider.class))
{
throw new QException("Core EMAIL provider does not support attachments; configure an attachment-capable messaging provider.");
}
}
QRecord saved = WatchRecords.get(DataAnalysisTables.SAVED, schedule.getValue("saved_id"));
String owner = saved.getValueString("owner_user_id");
SavedDefinition[] definition = new SavedDefinition[1];
SavedResult[] ownerResult = new SavedResult[1];
QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), session(owner)), () ->
{
definition[0] = SavedDefinition.parse(WatchRecords.get(DataAnalysisTables.SAVED, saved.getValue("id")));
if("TABLE_PROFILE".equals(definition[0].kind()) && definition[0].body().get("fields") == null)
{
var body = definition[0].body();
body.put("fields", service.capabilities(definition[0].request().getTableName()).getFields());
definition[0] = SavedDefinition.parse(definition[0].kind(), definition[0].request().getTableName(), body);
}
ownerResult[0] = definition[0].run(service, true);
});
List<String> skipped = new ArrayList<>();
List<String> failures = new ArrayList<>();
int sent = 0;
for(String recipient : ScheduleCustomizer.recipients(schedule))
{
SavedResult[] result = new SavedResult[1];
String address;
try
{
QSession viewer = session(recipient);
QContext.withTemporaryContext(new CapturedContext(QContext.getQInstance(), viewer), () ->
{
result[0] = recipient.equals(owner) ? ownerResult[0] : definition[0].run(service, true);
});
address = config.getRecipientAddressSupplier().apply(recipient);
if(address == null || address.isBlank())
{
throw new QException("No email address.");
}
}
catch(Exception e)
{
skipped.add(recipient);
continue;
}
try
{
SendMessageInput message = new SendMessageInput().withMessagingProviderName(config.getMessagingProviderName())
.withFrom(new Party().withAddress(config.getFromAddress())).withTo(new Party().withAddress(address))
.withSubject("Analysis: " + saved.getValueString("name"))
.withContent(new Content().withContentRole(EmailContentRole.HTML).withBody(html(saved.getValueString("name"), result[0])));
String format = schedule.getValueString("format");
if("CSV".equals(format) || "XLSX".equals(format))
{
var file = AnalysisExport.file(result[0], format);
message.withAttachment(new Attachment().withName(file.filename()).withContents(file.bytes()));
}
new SendMessageAction().execute(message);
sent++;
}
catch(Exception e)
{
failures.add(recipient);
}
}
String status = "Sent " + sent + "; skipped: " + (skipped.isEmpty() ? "none" : String.join(", ", skipped));
if(!failures.isEmpty())
{
throw new QException(status + "; delivery failed for: " + String.join(", ", failures));
}
return (status);
}
/*******************************************************************************
** Formatted result rows are escaped; pivots include only the first twenty rows.
*******************************************************************************/
public static String html(String name, SavedResult result)
{
var table = AnalysisExport.table(result);
StringBuilder html = new StringBuilder("<h2>").append(escape(name)).append("</h2>");
for(String warning : result.warnings())
{
html.append("<p>").append(escape(warning)).append("</p>");
}
html.append("<dl>");
for(var metric : AnalysisExport.headlines(result))
{
html.append("<dt>").append(escape(metric.getLabel())).append("</dt><dd>").append(escape(AnalysisExport.value(metric))).append("</dd>");
}
html.append("</dl><table><thead><tr>");
table.columns().forEach(c -> html.append("<th>").append(escape(c)).append("</th>"));
html.append("</tr></thead><tbody>");
for(var row : table.rows().stream().limit(20).toList())
{
html.append("<tr>");
row.forEach(value -> html.append("<td>").append(escape(value)).append("</td>"));
html.append("</tr>");
}
return (html.append("</tbody></table>").toString());
}
/*******************************************************************************
** Source labels and values are text, never trusted email markup.
*******************************************************************************/
private static String escape(String value)
{
return (Objects.toString(value, "").replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;").replace("'", "&#39;"));
}
}
@@ -0,0 +1,111 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis.schedule;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.temporal.TemporalAdjusters;
import java.util.Date;
import java.util.TimeZone;
import com.kingsrook.qqq.backend.core.exceptions.QBadRequestException;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import org.quartz.CronExpression;
/*******************************************************************************
** Calendar schedules use local wall time; cron uses Quartz's existing parser.
*******************************************************************************/
public final class ScheduleTiming
{
/*******************************************************************************
** Utility class.
*******************************************************************************/
private ScheduleTiming()
{
}
/*******************************************************************************
** Strictly after now; missing monthly dates clamp to that month's last day.
** Java resolves spring gaps forward and chooses the first autumn occurrence.
*******************************************************************************/
public static Instant next(QRecord row, Instant now) throws QBadRequestException
{
try
{
ZoneId zone = ZoneId.of(row.getValueString("time_zone") == null ? "UTC" : row.getValueString("time_zone"));
String cron = row.getValueString("cron");
if(cron != null && !cron.isBlank())
{
CronExpression expression = new CronExpression(cron);
expression.setTimeZone(TimeZone.getTimeZone(zone));
Date next = expression.getNextValidTimeAfter(Date.from(now));
if(next == null)
{
throw new IllegalArgumentException("Cron has no future occurrence.");
}
return (next.toInstant());
}
LocalTime time = LocalTime.parse(row.getValueString("time"));
String frequency = row.getValueString("frequency");
Integer day = row.getValueInteger("day");
LocalDate date = now.atZone(zone).toLocalDate();
for(int attempt = 0; attempt < 3; attempt++)
{
LocalDate candidate = switch(frequency)
{
case "DAILY" -> date;
case "WEEKLY" -> date.with(TemporalAdjusters.nextOrSame(java.time.DayOfWeek.of(day)));
case "MONTHLY" -> date.withDayOfMonth(Math.min(requireDay(day), date.lengthOfMonth()));
default -> throw new IllegalArgumentException("Choose DAILY, WEEKLY or MONTHLY.");
};
Instant instant = candidate.atTime(time).atZone(zone).toInstant();
if(instant.isAfter(now))
{
return (instant);
}
date = switch(frequency)
{
case "WEEKLY" -> candidate.plusWeeks(1);
case "MONTHLY" -> candidate.plusMonths(1).withDayOfMonth(1);
default -> candidate.plusDays(1);
};
}
throw new IllegalArgumentException("No future occurrence.");
}
catch(Exception e)
{
throw new QBadRequestException("Invalid schedule: choose a valid frequency/day, local time, time zone or Quartz cron.");
}
}
/*******************************************************************************
** Month days are one-based and never silently wrap into another month.
*******************************************************************************/
private static int requireDay(Integer day)
{
if(day == null || day < 1 || day > 31)
{
throw new IllegalArgumentException("Monthly day must be 1 to 31.");
}
return (day);
}
}