feat(saved): saved, shared, starred, scheduled and widget analyses, export, SQL handoff

Extension 1.4.0: saved analyses of every kind re-run as the viewer
(fields they cannot analyze are dropped), owner-or-shared READ through
core record locks with owner-only writes (and a database CHECK so a
private row can never lose its read owner), personal stars, schedules
that run as the owner and mail only recipients who could run the analysis
themselves, CSV/XLSX export, the SQL Workbench handoff, an analysis
dashboard widget, and an Analyses home built from standard widgets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
james.maesandClaude Opus 5.5 committed 2026-10-13 21:23:23 -05:00
1 parent 43c7571864
commit d2c9871d98
42 files changed
+3786 -296

No files matched your search

@@ -119,8 +119,17 @@ class DataAnalysisApiTest extends EngineFixture
*******************************************************************************/
private JsonNode call(HttpMethod method, String path, Object body, int status) throws Exception
{
String savedId = path.startsWith("/saved/") ? path.split("/")[2] : null;
if(savedId != null)
{
path = path.replace("/saved/" + savedId + "/", "/saved/{id}/");
}
var request = new UiApiRequest().withMethod(method).withPath(DataAnalysisApi.PREFIX + path)
.withPathParam("table", "orders").withSessionUUID(caller).withUserTimezone("America/Chicago");
if(savedId != null)
{
request.withPathParam("id", savedId);
}
if(body != null)
{
request.withJsonBody(body);
@@ -173,7 +182,7 @@ class DataAnalysisApiTest extends EngineFixture
assertThat(denied.getStatus()).isEqualTo(403);
assertThat(run.path(operation.equals("COLUMN_PROFILE") ? "columnProfile" : "tableProfile").isObject()).isTrue();
}
assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(14);
assertThat(invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList()).hasSize(21);
var client = java.net.http.HttpClient.newHttpClient();
var response = client.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create("http://127.0.0.1:" + server.getBoundPort() + "/qqq/v1/openapi.json")).build(), java.net.http.HttpResponse.BodyHandlers.ofString());
assertThat(response.statusCode()).isEqualTo(200);
@@ -355,7 +364,7 @@ class DataAnalysisApiTest extends EngineFixture
}
relationshipCountDrills(call(HttpMethod.POST, "/tables/{table}/crosstab", Map.of("rowField", "status", "columnField", "customer"), 200));
// All new schemas are discoverable from the same served OpenAPI catalog.
assertThat(com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi.extension().getSemanticVersion()).isEqualTo("1.3.0");
assertThat(com.kingsrook.qbits.dataanalysis.api.DataAnalysisApi.extension().getSemanticVersion()).isEqualTo("1.4.0");
}
/*******************************************************************************
@@ -452,4 +461,56 @@ class DataAnalysisApiTest extends EngineFixture
assertThat(invoker.invoke(request.withSessionUUID(caller).withPathParam("id", "99999")).getStatus()).isEqualTo(404);
assertThat(invoker.invoke(request.withPathParam("id", "invalid")).getStatus()).isEqualTo(400);
}
@Test
void savedOperationsAndFileResponsesOverHttp() throws Exception
{
var shared = TestSupport.insert(com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables.SAVED,
SavedAnalysesTest.definition("alice", true, "TABLE_PROFILE", Map.of("fields", List.of("amount", "secret"))));
var privateRow = TestSupport.insert(com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables.SAVED,
SavedAnalysesTest.definition("alice", false, "COLUMN_PROFILE", Map.of("fieldName", "amount")));
start();
caller = "watch-saved-viewer";
String path = "/saved/" + shared.getValue("id");
var result = call(HttpMethod.POST, path + "/run", null, 200);
assertThat(result.path("kind").asText()).isEqualTo("TABLE_PROFILE");
assertThat(result.path("warnings").toString()).contains("Some fields are not available to you");
assertThat(result.path("tableProfile").path("fields").size()).isEqualTo(1);
call(HttpMethod.POST, "/saved/" + privateRow.getValue("id") + "/run", null, 404);
call(HttpMethod.POST, "/saved/invalid/run", null, 400);
assertThat(call(HttpMethod.PUT, path + "/star", null, 200).path("starred").asBoolean()).isTrue();
call(HttpMethod.PUT, path + "/star", null, 200);
assertThat(call(HttpMethod.GET, "/saved", null, 200).path("analyses").get(0).path("starred").asBoolean()).isTrue();
assertThat(call(HttpMethod.DELETE, path + "/star", null, 200).path("starred").asBoolean()).isFalse();
assertThat(call(HttpMethod.POST, "/tables/{table}/sql", Map.of("kind", "TABLE_PROFILE", "request", Map.of("fields", List.of("amount"))), 200).path("available").asBoolean()).isFalse();
var client = java.net.http.HttpClient.newHttpClient();
for(String format : List.of("csv", "xlsx"))
{
String url = "http://127.0.0.1:" + server.getBoundPort() + "/qqq/v1" + DataAnalysisApi.PREFIX + path + "/export?format=" + format;
var response = client.send(java.net.http.HttpRequest.newBuilder(java.net.URI.create(url)).header("Cookie", "sessionUUID=" + caller).GET().build(), java.net.http.HttpResponse.BodyHandlers.ofByteArray());
assertThat(response.statusCode()).as(new String(response.body(), java.nio.charset.StandardCharsets.UTF_8)).isEqualTo(200);
assertThat(response.headers().firstValue("Content-Disposition").orElse("")).contains("analysis." + format);
if(format.equals("xlsx"))
{
try(var workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(new java.io.ByteArrayInputStream(response.body())))
{
assertThat(workbook.getSheetAt(0).getLastRowNum()).isPositive();
}
}
else
{
assertThat(new String(response.body(), java.nio.charset.StandardCharsets.UTF_8)).contains("Completeness");
}
}
var export = invoker.invoke(new UiApiRequest().withMethod(HttpMethod.POST).withPath(DataAnalysisApi.PREFIX + "/export").withSessionUUID(caller)
.withJsonBody(Map.of("kind", "COLUMN_PROFILE", "tableName", "orders", "request", Map.of("fieldName", "amount"), "format", "csv")));
assertThat(export.getStatus()).as(export.getBody()).isEqualTo(200);
assertThat(export.getBody()).contains("Counts");
call(HttpMethod.POST, "/export", Map.of("kind", "TABLE_PROFILE", "tableName", "orders", "request", Map.of(), "format", "pdf"), 400);
caller = "denied";
call(HttpMethod.POST, path + "/run", null, 403);
var operations = invoker.listOperations().stream().filter(o -> o.pathTemplate().startsWith(DataAnalysisApi.PREFIX)).toList();
assertThat(operations).hasSize(21);
assertThat(operations.stream().map(Object::toString).toList().toString()).contains("/saved/{id}/run", "/saved/{id}/star", "/saved/{id}/export", "/tables/{table}/sql", "/export");
}
}
@@ -20,131 +20,87 @@
package com.kingsrook.qbits.dataanalysis;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.List;
import com.kingsrook.qbits.dataanalysis.app.DataAnalysisHome;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.MultiStatisticsData;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.QInstance;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
/*******************************************************************************
** Real home counts, empty states and permission-scoped results.
*******************************************************************************/
class DataAnalysisHomeTest
{
/*******************************************************************************
** Release the thread-local context.
*******************************************************************************/
@AfterEach
void cleanup()
{
QContext.clear();
}
/*******************************************************************************
** Empty counts explain why; missing permissions are not represented as zero.
*******************************************************************************/
@Test
void emptyAndDenied() throws Exception
void standardWidgetsWithEmptyAndDeniedStates() throws Exception
{
QInstance instance = TestSupport.instance();
var instance = TestSupport.instance();
QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read"));
String html = DataAnalysisHome.render(Instant.now());
assertThat(html).contains("Mine: 0", "Starred by me: 0", "No saved analyses yet",
"No watches yet.", "No analysis runs yet.", "View all", "Open any table and choose Analyze on a column header to explore its data.", "<h3>");
var widget = new DataAnalysisHome().render(new RenderWidgetInput()).getWidgetData();
assertThat(widget).isInstanceOf(com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML.class);
assertThat(((com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML) widget).getType()).isEqualTo("html");
assertThat(html).contains("&middot;").doesNotContain("Â", "later", "phase", "<table", "color:", "width:360", "width: 360");
assertThat(html.chars().allMatch(c -> c < 128)).isTrue();
QContext.init(instance, TestSupport.user("alice"));
html = DataAnalysisHome.render(Instant.now());
assertThat(html).contains("you need data_analysis.saved.read", "you need data_analysis.watch.read").doesNotContain("Mine: 0", "Complete: 0");
QContext.init(instance, new QSession());
assertThat(DataAnalysisHome.render(Instant.now())).contains("Sign in to see your saved analyses");
}
/*******************************************************************************
** Different owners, shared flags, stars, disabled watches and date boundaries.
*******************************************************************************/
@Test
void countsUseCurrentUserAndRealRows() throws Exception
{
QInstance instance = TestSupport.instance();
Instant now = Instant.parse("2026-10-13T12:00:00Z");
QRecord mine = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", true));
TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false));
QRecord bob = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("bob", true));
TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", mine.getValue("id")).withValue("user_id", "alice"));
TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", bob.getValue("id")).withValue("user_id", "alice"));
TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", mine.getValue("id")).withValue("user_id", "bob"));
TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("SPIKE", "COUNT", null).withValue("enabled", true).withValue("state", "FIRING"));
TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("DROP", "COUNT", null).withValue("enabled", true));
TestSupport.insert(DataAnalysisTables.WATCH, WatchDetectorsTest.definition("DROP", "COUNT", null).withValue("enabled", false));
run("alice", "COMPLETE", now);
run("alice", "COMPLETE", now.minus(7, ChronoUnit.DAYS));
run("alice", "FAILED", now.minus(1, ChronoUnit.DAYS));
run("alice", "COMPLETE", now.minus(7, ChronoUnit.DAYS).minusSeconds(1));
run("alice", "FAILED", now.plusSeconds(1));
run("alice", "RUNNING", now);
run("bob", "FAILED", now);
QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read"));
String html = DataAnalysisHome.render(now);
assertThat(html).contains("Mine: 2", "Shared (visible to you): 1", "Starred by me: 1", "Enabled: 2", "Firing: 1", "Recent analysis runs", "Complete", "Failed", "Running", "Not finished")
.doesNotContain("No saved analyses", "No watches yet", "No analysis runs yet");
QContext.init(instance, TestSupport.user("bob").withPermissions("data_analysis.saved.read", "data_analysis.run.read"));
html = DataAnalysisHome.render(now);
assertThat(html).contains("Mine: 1", "Shared (visible to you): 1", "Starred by me: 0", "Failed", "you need data_analysis.watch.read");
}
/*******************************************************************************
** Latest five remain owner-scoped even with administrator bypass and escape HTML.
*******************************************************************************/
@Test
void recentRunDetailsAndLimit() throws Exception
{
QInstance instance = TestSupport.instance();
instance.getTable(DataAnalysisTables.SAVED).withLabel("Saved <analysis> & review");
Instant now = Instant.parse("2026-10-13T23:10:00Z");
for(int i = 0; i < 7; i++)
var counts = DataAnalysisHome.statistics(Instant.now());
assertThat(counts.getStatisticsGroupData()).flatExtracting(g -> g.getStatisticList()).allSatisfy(s -> assertThat(s.getValue()).isZero());
for(var widget : DataAnalysisHome.widgets())
{
TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", DataAnalysisTables.SAVED)
.withValue("operation", "COLUMN_PROFILE").withValue("user_id", "admin").withValue("status", "COMPLETE")
.withValue("started_at", now.minusSeconds(i * 60L)).withValue("finished_at", now.minusSeconds(i * 60L)));
assertThat(widget.getType()).isIn("multiStatistics", "table");
var data = new DataAnalysisHome().render(new RenderWidgetInput().withWidgetMetaData(widget)).getWidgetData();
if(data instanceof TableData table)
{
assertThat(table.getColumns()).hasSizeLessThanOrEqualTo(4);
assertThat(table.getRows()).isEmpty();
assertThat(table.getNoRowsFoundHTML()).startsWith("No ");
}
}
TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "private_bob_table")
.withValue("operation", "TABLE_PROFILE").withValue("user_id", "bob").withValue("status", "FAILED").withValue("started_at", now));
QContext.init(instance, TestSupport.admin().withValue(QSession.VALUE_KEY_USER_TIMEZONE, "America/Chicago"));
String html = DataAnalysisHome.render(now);
assertThat(html).contains("Saved &lt;analysis&gt; &amp; review", "Column Profile", "2026-10-13 18:10", "2026-10-13 18:06", "<ul", "<dl")
.doesNotContain("private_bob_table", "2026-10-13 18:05", "<analysis>");
assertThat(html.split("<li>", -1)).hasSize(6);
assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "/1\"");
assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "\">View all</a>");
QContext.init(instance, TestSupport.user("alice"));
assertThat(DataAnalysisHome.statistics(Instant.now()).getStatisticsGroupData()).allSatisfy(g ->
{
assertThat(g.getStatisticList()).isEmpty();
assertThat(g.getSubheader()).contains("Unavailable");
});
assertThat(((TableData) DataAnalysisHome.render(DataAnalysisHome.FIRING, Instant.now())).getNoRowsFoundHTML()).contains("Unavailable");
}
@Test
void countsRowsLinksAndDateBoundaries() throws Exception
{
var instance = TestSupport.instance();
Instant now = Instant.parse("2026-10-13T12:00:00Z");
var mine = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false).withValue("name", "<script>& profile"));
var shared = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("bob", true));
var hidden = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("bob", false));
for(var saved : List.of(mine, shared, hidden))
{
TestSupport.insert(DataAnalysisTables.SAVED_STAR, new QRecord().withValue("saved_id", saved.getValue("id")).withValue("user_id", "alice"));
}
for(Instant start : List.of(now, now.minus(7, ChronoUnit.DAYS), now.minus(7, ChronoUnit.DAYS).minusSeconds(1), now.plusSeconds(1)))
{
run("alice", start);
}
run("bob", now);
QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.run.read"));
var counts = DataAnalysisHome.statistics(now).getStatisticsGroupData();
assertThat(counts.getFirst().getStatisticList()).extracting(s -> s.getValue()).containsExactly(1, 1, 2);
assertThat(counts.getLast().getStatisticList().getFirst().getValue()).isEqualTo(2);
TableData starred = (TableData) DataAnalysisHome.render(DataAnalysisHome.STARRED, now);
assertThat(starred.getRows()).hasSize(2);
assertThat(starred.getRows().toString()).contains("&lt;script&gt;&amp; profile", instance.getTablePath(DataAnalysisTables.SAVED)).doesNotContain("<script>");
TableData recent = (TableData) DataAnalysisHome.render(DataAnalysisHome.RECENT, now);
assertThat(recent.getRows()).hasSize(3);
assertThat(recent.getLinkURL()).isEqualTo(instance.getTablePath(DataAnalysisTables.RUN));
}
/*******************************************************************************
** Insert a real run row with an explicit owner and start time.
*******************************************************************************/
private static void run(String user, String status, Instant started) throws Exception
private static void run(String user, Instant started) throws Exception
{
TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "orders").withValue("operation", "COLUMN_PROFILE")
.withValue("user_id", user).withValue("status", status).withValue("started_at", started));
.withValue("user_id", user).withValue("status", "COMPLETE").withValue("started_at", started));
}
}
@@ -69,7 +69,7 @@ class DataAnalysisMetadataTest
assertThat(instance.getTables()).containsOnlyKeys(DataAnalysisTables.ALL);
assertThat(instance.getPossibleValueSources().keySet().stream().filter(name -> name.startsWith("data_analysis_"))).hasSize(16);
assertThat(instance.getJoins()).hasSize(3);
assertThat(instance.getWidgets()).hasSize(4).containsKey(DataAnalysisHome.HOME_WIDGET);
assertThat(instance.getWidgets()).hasSize(8).containsKey(DataAnalysisHome.HOME_WIDGET);
assertThat(instance.getApp("dataAnalysis").getSections().getFirst().getProcesses()).isNullOrEmpty();
assertThat(instance.getApp("dataAnalysis").getLabel()).isEqualTo("Analyses");
assertThat(instance.getApp("dataAnalysis").getSections().getFirst().getTables()).containsExactlyElementsOf(DataAnalysisTables.NAV_TABLES);
@@ -90,7 +90,7 @@ class DataAnalysisMetadataTest
assertThat(lock.getSecurityKeyType()).isEqualTo("data_analysis_user");
assertThat(lock.getFieldName()).isEqualTo(name.equals(DataAnalysisTables.SAVED) ? "owner_user_id" : "user_id");
assertThat(lock.getNullValueBehavior()).isEqualTo(RecordSecurityLock.NullValueBehavior.DENY);
assertThat(lock.getLockScope()).isEqualTo(RecordSecurityLock.LockScope.READ_AND_WRITE);
assertThat(lock.getLockScope()).isEqualTo(name.equals(DataAnalysisTables.SAVED) ? RecordSecurityLock.LockScope.WRITE : RecordSecurityLock.LockScope.READ_AND_WRITE);
}
assertThat(instance.getSecurityKeyType("data_analysis_user").getAllAccessKeyName()).isEqualTo("data_analysis.admin");
assertThat(PermissionsHelper.getAllAvailablePermissionNames(instance)).contains("data_analysis.settings.read", "data_analysis.settings.edit", "data_analysis.run.read", "data_analysis.run.delete", "data_analysis.saved.read", "data_analysis.saved.insert",
@@ -205,7 +205,8 @@ class DataAnalysisMetadataTest
assertThat(ddl).contains("id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY", "create_date TIMESTAMP WITHOUT TIME ZONE",
"modify_date TIMESTAMP WITHOUT TIME ZONE", "rows_scanned BIGINT", "result_json TEXT", "shared BOOLEAN", "day INTEGER",
"owner_user_id VARCHAR(250) NOT NULL", "date_field VARCHAR(250) NOT NULL", "last_value NUMERIC", "last_message TEXT", "CREATE UNIQUE INDEX IF NOT EXISTS uq_data_analysis_table_config_1 ON data_analysis_table_config (table_name)",
"CREATE UNIQUE INDEX IF NOT EXISTS uq_data_analysis_saved_star_1 ON data_analysis_saved_star (saved_id, user_id)");
"CREATE UNIQUE INDEX IF NOT EXISTS uq_data_analysis_saved_star_1 ON data_analysis_saved_star (saved_id, user_id)",
"CHECK (shared = TRUE OR read_owner_user_id IS NOT NULL)");
assertThat(ddl).endsWith(";\n");
}
}
@@ -67,9 +67,13 @@ class DataAnalysisPossibleValuesTest
void watchSavedAnalysisDependsOnTable() throws Exception
{
QInstance instance = TestSupport.instance();
QRecord orders = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false));
for(String name : List.of("orders", "customers"))
{
instance.addTable(new QTableMetaData().withName(name).withField(new QFieldMetaData("table_name", QFieldType.STRING)));
}
QRecord orders = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false).withValue("table_name", "orders"));
QRecord customers = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false).withValue("table_name", "customers"));
TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("bob", true));
TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("bob", false).withValue("table_name", "orders"));
QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.edit"));
QFieldMetaData field = instance.getTable(DataAnalysisTables.WATCH).getField("saved_id");
assertThat(search(field, Map.of("table_name", "orders"))).extracting(value -> (Object) value.getId()).containsExactly(orders.getValueInteger("id"));
@@ -125,7 +125,7 @@ class DataAnalysisSecurityTest
}
/*******************************************************************************
** Saved, star and run locks hide others even when the shared flag is true.
** Shared saved rows are readable; private stars, schedules and runs remain owner-locked.
*******************************************************************************/
@Test
void ownerIsolation() throws Exception
@@ -142,12 +142,12 @@ class DataAnalysisSecurityTest
TestSupport.insert(DataAnalysisTables.SCHEDULE, new QRecord().withValue("saved_id", mine.getValue("id")));
TestSupport.insert(DataAnalysisTables.SCHEDULE, new QRecord().withValue("saved_id", other.getValue("id")));
QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.run.read"));
assertThat(query(DataAnalysisTables.SAVED)).extracting(r -> r.getValueString("owner_user_id")).containsExactly("alice");
assertThat(query(DataAnalysisTables.SAVED)).extracting(r -> r.getValueString("owner_user_id")).containsExactly("alice", "bob");
assertThat(query(DataAnalysisTables.SAVED_STAR)).extracting(r -> r.getValueString("user_id")).containsExactly("alice");
assertThat(query(DataAnalysisTables.RUN)).extracting(r -> r.getValueString("user_id")).containsExactly("alice");
assertThat(query(DataAnalysisTables.SCHEDULE)).extracting(r -> r.getValue("saved_id")).containsExactly(mine.getValue("id"));
QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.admin"));
assertThat(query(DataAnalysisTables.SAVED)).hasSize(1);
assertThat(query(DataAnalysisTables.SAVED)).hasSize(2);
QContext.getQSession().withSecurityKeyValue("data_analysis.admin", true);
assertThat(query(DataAnalysisTables.SAVED)).hasSize(2);
}
@@ -0,0 +1,195 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis;
import java.io.ByteArrayInputStream;
import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.dataanalysis.app.SavedAnalysisWidget;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.saved.AnalysisExport;
import com.kingsrook.qbits.dataanalysis.saved.SavedAnalyses;
import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition;
import com.kingsrook.qbits.dataanalysis.saved.SqlHandoff;
import com.kingsrook.qbits.dataanalysis.watch.WatchRecords;
import com.kingsrook.qqq.backend.core.actions.tables.InsertAction;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput;
import com.kingsrook.qqq.backend.core.model.dashboard.widgets.ChartData;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.utils.JsonUtils;
import org.apache.poi.ss.usermodel.WorkbookFactory;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
class SavedAnalysesTest extends EngineFixture
{
static QSession viewer(String id, int customer)
{
return (TestSupport.user(id).withPermissions("data_analysis.saved.read", "data_analysis.saved.insert", "data_analysis.saved.edit", "data_analysis.saved.delete",
"orders.read", "orders.analyze", "customer.read").withSecurityKeyValue("customer_scope", customer));
}
static QRecord definition(String owner, boolean shared, String kind, Map<String, Object> body)
{
return (TestSupport.saved(owner, shared).withValue("table_name", "orders").withValue("kind", kind).withValue("definition_json", JsonUtils.toJson(body)));
}
@Test
void sharedReadOwnerWriteAndFieldReduction() throws Exception
{
var saved = TestSupport.insert(DataAnalysisTables.SAVED, definition("alice", true, "TABLE_PROFILE", Map.of("fields", List.of("amount", "secret"))));
var privateRow = TestSupport.insert(DataAnalysisTables.SAVED, definition("alice", false, "COLUMN_PROFILE", Map.of("fieldName", "amount")));
QContext.init(instance, viewer("bob", 2));
var api = new SavedAnalyses(service);
assertThat(api.list()).extracting(SavedAnalyses.Entry::id).contains(saved.getValue("id")).doesNotContain(privateRow.getValue("id"));
var result = api.run(saved.getValue("id"));
assertThat(result.kind()).isEqualTo("TABLE_PROFILE");
assertThat(result.warnings()).containsExactly(SavedDefinition.FIELD_WARNING);
assertThat(result.tableProfile().getFields()).extracting(f -> f.getField()).containsExactly("amount");
assertThat(((Number) result.tableProfile().getRowCount().getValue()).intValue()).isEqualTo(4);
assertThatThrownBy(() -> api.run(privateRow.getValue("id"))).isInstanceOf(QException.class);
var patch = new QRecord().withValue("id", saved.getValue("id")).withValue("name", "Stolen").withValue("read_owner_user_id", "bob");
assertThat(new UpdateAction().execute(new UpdateInput(DataAnalysisTables.SAVED).withRecords(List.of(patch))).getRecords().getFirst().getErrors()).isNotEmpty();
var deleted = new com.kingsrook.qqq.backend.core.actions.tables.DeleteAction().execute(
new com.kingsrook.qqq.backend.core.model.actions.tables.delete.DeleteInput(DataAnalysisTables.SAVED).withPrimaryKeys(List.of(saved.getValue("id"))));
assertThat(deleted.getDeletedRecordCount()).isZero();
QContext.init(instance, viewer("alice", 1));
var updated = new UpdateAction().execute(new UpdateInput(DataAnalysisTables.SAVED).withRecords(List.of(new QRecord().withValue("id", saved.getValue("id")).withValue("shared", false)))).getRecords().getFirst();
assertThat(updated.getErrors()).isNullOrEmpty();
QContext.init(instance, viewer("bob", 2));
assertThat(api.list()).isEmpty();
}
@Test
void starsAreIdempotentAndPersonal() throws Exception
{
var saved = TestSupport.insert(DataAnalysisTables.SAVED, definition("alice", true, "COLUMN_PROFILE", Map.of("fieldName", "amount")));
QContext.init(instance, viewer("bob", 2));
var api = new SavedAnalyses(service);
assertThat(api.star(saved.getValue("id"), true)).isTrue();
api.star(saved.getValue("id"), true);
assertThat(api.list().getFirst().starred()).isTrue();
QContext.init(instance, viewer("alice", 1));
assertThat(api.list().getFirst().starred()).isFalse();
QContext.init(instance, viewer("bob", 2));
assertThat(api.star(saved.getValue("id"), false)).isFalse();
api.star(saved.getValue("id"), false);
assertThat(api.list().getFirst().starred()).isFalse();
}
@Test
void malformedDefinitionsAndPartialEditsReturnPlainErrors() throws Exception
{
for(String json : List.of("not json", "[]", "{}", "{\"fieldName\":\"missing\"}", "{\"tableName\":\"other\",\"fieldName\":\"amount\"}"))
{
QRecord row = definition("admin", false, "COLUMN_PROFILE", Map.of()).withValue("definition_json", json);
var inserted = new InsertAction().execute(new InsertInput(DataAnalysisTables.SAVED).withRecords(List.of(row))).getRecords().getFirst();
assertThat(inserted.getErrors()).isNotEmpty();
}
assertThatThrownBy(() -> SavedDefinition.parse("UNKNOWN", "orders", Map.of())).hasMessageContaining("kind");
assertThatThrownBy(() -> SavedDefinition.parse("TABLE_PROFILE", "unknown", Map.of())).hasMessageContaining("table");
var saved = TestSupport.insert(DataAnalysisTables.SAVED, definition("admin", false, "COLUMN_PROFILE", Map.of("fieldName", "amount")));
var invalid = new QRecord().withValue("id", saved.getValue("id")).withValue("definition_json", "{\"fieldName\":\"nope\"}");
assertThat(new UpdateAction().execute(new UpdateInput(DataAnalysisTables.SAVED).withRecords(List.of(invalid))).getRecords().getFirst().getErrors()).isNotEmpty();
assertThat(WatchRecords.get(DataAnalysisTables.SAVED, saved.getValue("id")).getValueString("definition_json")).contains("amount");
}
@Test
void allKindsRunAndExportRealTables() throws Exception
{
Map<String, Map<String, Object>> definitions = Map.of(
"COLUMN_PROFILE", Map.of("fieldName", "amount"),
"TABLE_PROFILE", Map.of("fields", List.of("amount", "status")),
"PIVOT", Map.of("rows", List.of(Map.of("field", "status")), "values", List.of(Map.of("operator", "COUNT"))),
"COMPARE", Map.of("fields", List.of("amount", "status")),
"CORRELATION", Map.of("fields", List.of("amount", "quantity")),
"CROSSTAB", Map.of("rowField", "status", "columnField", "paid"),
"SHIFT", Map.of("field", "amount", "dateField", "created", "bucket", "DAY"));
for(var entry : definitions.entrySet())
{
var saved = TestSupport.insert(DataAnalysisTables.SAVED, definition("alice", true, entry.getKey(), entry.getValue()));
QContext.init(instance, viewer("alice", 1));
var result = new SavedAnalyses(service).run(saved.getValue("id"));
assertThat(result.kind()).isEqualTo(entry.getKey());
assertThat(AnalysisExport.table(result).rows()).as(entry.getKey()).isNotEmpty();
var csv = AnalysisExport.file(result, "csv");
assertThat(new String(csv.bytes(), StandardCharsets.UTF_8)).contains(AnalysisExport.table(result).columns().getFirst());
var xlsx = AnalysisExport.file(result, "xlsx");
try(var workbook = WorkbookFactory.create(new ByteArrayInputStream(xlsx.bytes())))
{
var sheet = workbook.getSheetAt(0);
assertThat(sheet.getLastRowNum()).isEqualTo(AnalysisExport.table(result).rows().size());
assertThat(sheet.getRow(0).getCell(0).getStringCellValue()).isEqualTo(AnalysisExport.table(result).columns().getFirst());
assertThat(sheet.getRow(1).getCell(0).getStringCellValue()).isEqualTo(AnalysisExport.table(result).rows().getFirst().getFirst());
}
var widget = SavedAnalysisWidget.widget().withDefaultValue("savedId", saved.getValue("id"));
var chart = (ChartData) new SavedAnalysisWidget().render(new RenderWidgetInput().withWidgetMetaData(widget)).getWidgetData();
assertThat(chart.getChartData().getDatasets()).as(entry.getKey()).isNotEmpty();
assertThat(chart.getFooterHTML()).contains("View saved analysis");
QContext.init(instance, session());
}
assertThatThrownBy(() -> AnalysisExport.format("pdf")).hasMessageContaining("csv or xlsx");
}
@Test
void unavailableRequiredFieldsAndFiltersNeverBroadenResults() throws Exception
{
QContext.init(instance, viewer("alice", 1));
for(var definition : List.of(
SavedDefinition.parse("COLUMN_PROFILE", "orders", Map.of("fieldName", "secret")),
SavedDefinition.parse("CORRELATION", "orders", Map.of("fields", List.of("amount", "secret"))),
SavedDefinition.parse("TABLE_PROFILE", "orders", Map.of("fields", List.of("amount"), "filter", Map.of("criteria", List.of(Map.of("fieldName", "secret", "operator", "EQUALS", "values", List.of("x"))))))))
{
var result = definition.run(service, false);
assertThat(result.warnings()).contains(SavedDefinition.FIELD_WARNING);
assertThat(AnalysisExport.table(result).rows()).isEmpty();
assertThatThrownBy(() -> definition.run(service, true)).hasMessageContaining(SavedDefinition.FIELD_WARNING);
}
}
@Test
void savedWidgetUsesViewerAndMemoryHasNoSql() throws Exception
{
var saved = TestSupport.insert(DataAnalysisTables.SAVED, definition("alice", true, "COLUMN_PROFILE", Map.of("fieldName", "amount")));
QContext.init(instance, viewer("bob", 2));
var empty = (ChartData) new SavedAnalysisWidget().render(new RenderWidgetInput().withWidgetMetaData(SavedAnalysisWidget.widget())).getWidgetData();
assertThat(empty.getDescription()).contains("Choose a saved analysis");
var metadata = SavedAnalysisWidget.widget().withDefaultValue("savedId", saved.getValue("id"));
var data = (ChartData) new SavedAnalysisWidget().render(new RenderWidgetInput().withWidgetMetaData(metadata)).getWidgetData();
assertThat(data.getChartData().getLabels()).isNotEmpty();
assertThat(data.getDescription()).contains("4");
assertThat(data.getFooterHTML()).contains("/" + saved.getValue("id"));
var definition = SavedDefinition.parse("COLUMN_PROFILE", "orders", Map.of("fieldName", "amount"));
assertThat(SqlHandoff.execute(definition, service).available()).isFalse();
instance.addApp(new QAppMetaData().withName("sqlWorkbench"));
QContext.getQSession().withPermission("sql_workbench.hasAccess");
assertThat(SqlHandoff.execute(definition, service).note()).contains("non-RDBMS");
}
}
@@ -0,0 +1,72 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition;
import com.kingsrook.qbits.dataanalysis.saved.SqlHandoff;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData;
import com.kingsrook.qqq.backend.module.rdbms.model.metadata.RDBMSBackendMetaData;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
class SavedSqlH2Test extends EngineFixture
{
@Override
protected RDBMSBackendMetaData database()
{
return (new RDBMSBackendMetaData().withVendor("h2").withUsername("sa").withPassword("")
.withJdbcUrl("jdbc:h2:mem:saved_" + UUID.randomUUID() + ";MODE=PostgreSQL;DB_CLOSE_DELAY=-1;DATABASE_TO_UPPER=FALSE"));
}
@Test
void actualMainAggregateHasSecurityFiltersAndWorkbenchPermission() throws Exception
{
instance.addApp(new QAppMetaData().withName("sqlWorkbench"));
QContext.init(instance, SavedAnalysesTest.viewer("alice", 1));
var definition = SavedDefinition.parse("PIVOT", "orders", Map.of("rows", List.of(Map.of("field", "status")), "values", List.of(Map.of("operator", "COUNT"))));
assertThat(SqlHandoff.execute(definition, service).available()).isFalse();
QContext.getQSession().withPermission("sql_workbench.hasAccess");
var result = SqlHandoff.execute(definition, service);
assertThat(result.available()).as(result.note()).isTrue();
assertThat(result.sql().toLowerCase()).contains("select", "count(", "group by", "customer", "status");
assertThat(result.note()).contains("Record security", "for reading");
}
@Test
void profileMainStatementIsTheMetricBatchAndCaptureDoesNotMutateMetadata() throws Exception
{
instance.addApp(new QAppMetaData().withName("sqlWorkbench"));
QContext.init(instance, SavedAnalysesTest.viewer("alice", 1).withPermission("sql_workbench.hasAccess"));
var before = java.util.Set.copyOf(instance.getTable("orders").getEnabledCapabilities());
var result = SqlHandoff.execute(SavedDefinition.parse("COLUMN_PROFILE", "orders", Map.of("fieldName", "amount")), service);
assertThat(result.available()).as(result.note()).isTrue();
assertThat(result.sql().toLowerCase()).contains("select", "amount", "customer").doesNotContain("group by");
assertThat(instance.getTable("orders").getEnabledCapabilities()).containsExactlyInAnyOrderElementsOf(before);
// A second request captures its own SQL rather than stale thread-local state.
var next = SqlHandoff.execute(SavedDefinition.parse("CORRELATION", "orders", Map.of("fields", List.of("amount", "quantity"))), service);
assertThat(next.available()).as(next.note()).isTrue();
assertThat(next.sql().toLowerCase()).contains("corr(");
}
}
@@ -0,0 +1,231 @@
/*
* QQQ - Low-code Application Framework for Engineers.
* Copyright (C) 2021-2025. Kingsrook, LLC
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
* contact@kingsrook.com
* https://github.com/Kingsrook/
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.kingsrook.qbits.dataanalysis;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables;
import com.kingsrook.qbits.dataanalysis.processes.ScheduleActions;
import com.kingsrook.qbits.dataanalysis.saved.AnalysisExport;
import com.kingsrook.qbits.dataanalysis.saved.SavedDefinition;
import com.kingsrook.qbits.dataanalysis.schedule.ScheduleRunner;
import com.kingsrook.qbits.dataanalysis.schedule.ScheduleTiming;
import com.kingsrook.qbits.dataanalysis.watch.WatchRecords;
import com.kingsrook.qqq.backend.core.actions.tables.UpdateAction;
import com.kingsrook.qqq.backend.core.alerts.AlertsInstanceMetaData;
import com.kingsrook.qqq.backend.core.context.QContext;
import com.kingsrook.qqq.backend.core.exceptions.QException;
import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageInput;
import com.kingsrook.qqq.backend.core.model.actions.messaging.SendMessageOutput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepInput;
import com.kingsrook.qqq.backend.core.model.actions.processes.RunBackendStepOutput;
import com.kingsrook.qqq.backend.core.model.actions.tables.update.UpdateInput;
import com.kingsrook.qqq.backend.core.model.data.QRecord;
import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference;
import com.kingsrook.qqq.backend.core.model.metadata.messaging.QMessagingProviderMetaData;
import com.kingsrook.qqq.backend.core.model.session.QSession;
import com.kingsrook.qqq.backend.core.modules.messaging.MessagingProviderInterface;
import com.kingsrook.qqq.backend.core.modules.messaging.QMessagingProviderDispatcher;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
class SchedulesTest extends EngineFixture
{
static final Instant NOW = Instant.parse("2026-10-13T12:00:00Z");
private ScheduleRunner runner;
public static class Mail implements MessagingProviderInterface
{
static final List<SendMessageInput> sent = new ArrayList<>();
static boolean fail;
@Override
public String getType()
{
return ("analysis-test-mail");
}
@Override
public SendMessageOutput sendMessage(SendMessageInput input) throws QException
{
if(fail)
{
throw new QException("Test mail failure");
}
sent.add(input);
return (new SendMessageOutput());
}
}
@BeforeEach
void deliverySetup() throws Exception
{
Mail.sent.clear();
Mail.fail = false;
WatchesTest.Sink.raised.clear();
QMessagingProviderDispatcher.registerMessagingProvider(new Mail());
instance.addMessagingProvider(new QMessagingProviderMetaData().withName("mail").withType("analysis-test-mail"));
AlertsInstanceMetaData.ofOrWithNew(instance).withSink(new QCodeReference(WatchesTest.Sink.class));
config.withMessagingProviderName("mail").withFromAddress("analysis@example.test").withRecipientAddressSupplier(id -> id + "@example.test")
.withOwnerSessionSupplier(id ->
{
if(id.equals("missing"))
{
return (null);
}
QSession user = SavedAnalysesTest.viewer(id, id.equals("alice") ? 1 : 2);
if(id.equals("denied"))
{
user.getPermissions().remove("orders.analyze");
}
return (user);
});
runner = new ScheduleRunner(config, service);
}
QRecord schedule(String kind, Map<String, Object> definition, String format) throws Exception
{
var saved = TestSupport.insert(DataAnalysisTables.SAVED, SavedAnalysesTest.definition("alice", false, kind, definition).withValue("name", "<Orders>"));
var schedule = TestSupport.insert(DataAnalysisTables.SCHEDULE, new QRecord().withValue("saved_id", saved.getValue("id"))
.withValue("frequency", "DAILY").withValue("time", "09:00").withValue("time_zone", "America/Chicago")
.withValue("recipients", "[\"alice\",\"bob\",\"denied\",\"missing\"]").withValue("format", format).withValue("enabled", true));
new UpdateAction().execute(new UpdateInput(DataAnalysisTables.SCHEDULE).withRecords(List.of(new QRecord().withValue("id", schedule.getValue("id")).withValue("next_run_at", NOW))));
return (schedule);
}
@Test
void calendarWeeklyMonthlyDstAndCron() throws Exception
{
QRecord row = new QRecord().withValue("frequency", "DAILY").withValue("time", "09:00").withValue("time_zone", "America/Chicago");
assertThat(ScheduleTiming.next(row, NOW)).isEqualTo(Instant.parse("2026-10-13T14:00:00Z"));
row.withValue("frequency", "WEEKLY").withValue("day", 1);
assertThat(ScheduleTiming.next(row, NOW)).isEqualTo(Instant.parse("2026-10-19T14:00:00Z"));
row.withValue("frequency", "MONTHLY").withValue("day", 31);
assertThat(ScheduleTiming.next(row, Instant.parse("2026-02-01T00:00:00Z"))).isEqualTo(Instant.parse("2026-02-28T15:00:00Z"));
row.withValue("frequency", "DAILY").withValue("time", "02:30");
assertThat(ScheduleTiming.next(row, Instant.parse("2026-03-08T06:00:00Z"))).isEqualTo(Instant.parse("2026-03-08T08:30:00Z"));
row.withValue("time", "01:30");
assertThat(ScheduleTiming.next(row, Instant.parse("2026-11-01T07:00:00Z"))).isEqualTo(Instant.parse("2026-11-02T07:30:00Z"));
row.withValue("cron", "0 0 9 ? * MON");
assertThat(ScheduleTiming.next(row, NOW)).isEqualTo(Instant.parse("2026-10-19T14:00:00Z"));
row.withValue("cron", "bad cron");
assertThatThrownBy(() -> ScheduleTiming.next(row, NOW)).hasMessageContaining("Invalid schedule");
row.withValue("cron", null).withValue("time_zone", "Mars/Time");
assertThatThrownBy(() -> ScheduleTiming.next(row, NOW)).hasMessageContaining("Invalid schedule");
row.withValue("time_zone", "UTC").withValue("frequency", "MONTHLY").withValue("day", 32);
assertThatThrownBy(() -> ScheduleTiming.next(row, NOW)).hasMessageContaining("Invalid schedule");
}
@Test
void dueRunsAsOwnerAndRecipientRowsAreRecheckedWithAttachments() throws Exception
{
var schedule = schedule("PIVOT", Map.of("rows", List.of(Map.of("field", "status")), "values", List.of(Map.of("operator", "COUNT"))), "CSV");
assertThat(runner.runDue(NOW.minusSeconds(1))).isZero();
assertThat(runner.runDue(NOW)).isEqualTo(1);
assertThat(Mail.sent).hasSize(2);
assertThat(Mail.sent).extracting(m -> m.getTo().getAddress()).containsExactly("alice@example.test", "bob@example.test");
String ownerCsv = new String(Mail.sent.get(0).getAttachmentList().getFirst().getContents(), StandardCharsets.UTF_8);
String viewerCsv = new String(Mail.sent.get(1).getAttachmentList().getFirst().getContents(), StandardCharsets.UTF_8);
assertThat(ownerCsv).contains("4");
assertThat(viewerCsv).contains("2").doesNotContain("4");
assertThat(Mail.sent.getFirst().getContentList().getFirst().getBody()).contains("&lt;Orders&gt;", "<table>").doesNotContain("<Orders>");
var stored = WatchRecords.get(DataAnalysisTables.SCHEDULE, schedule.getValue("id"));
assertThat(stored.getValueString("last_status")).contains("Sent 2", "denied", "missing");
assertThat(stored.getValueInstant("last_run_at")).isEqualTo(NOW);
assertThat(stored.getValueInstant("next_run_at")).isAfter(NOW);
assertThat(runner.runDue(NOW)).isZero();
assertThat(WatchesTest.Sink.raised).isEmpty();
}
@Test
void providerFailureAndMissingOwnerRaiseVisibleAlerts() throws Exception
{
var schedule = schedule("COLUMN_PROFILE", Map.of("fieldName", "amount"), "XLSX");
config.withMessagingProviderName(null);
assertThat(runner.sendNow(schedule.getValue("id"), NOW)).contains("No messaging provider configured");
assertThat(WatchesTest.Sink.raised).hasSize(1);
assertThat(WatchesTest.Sink.raised.getFirst().getAlertTypeName()).isEqualTo(ScheduleRunner.ALERT_TYPE);
config.withMessagingProviderName("mail").withOwnerSessionSupplier(id -> null);
assertThat(runner.sendNow(schedule.getValue("id"), NOW)).contains("User session unavailable");
}
@Test
void xlsxDeliveryFieldDenialAndManualReviewResults() throws Exception
{
var schedule = schedule("COLUMN_PROFILE", Map.of("fieldName", "amount"), "XLSX");
assertThat(runner.sendNow(schedule.getValue("id"), NOW)).contains("Sent 2");
var attachment = Mail.sent.getFirst().getAttachmentList().getFirst();
assertThat(attachment.getName()).isEqualTo("analysis.xlsx");
try(var workbook = org.apache.poi.ss.usermodel.WorkbookFactory.create(new java.io.ByteArrayInputStream(attachment.getContents())))
{
assertThat(workbook.getSheetAt(0).getLastRowNum()).isGreaterThan(5);
}
Mail.fail = true;
assertThat(runner.sendNow(schedule.getValue("id"), NOW)).contains("delivery failed for: alice, bob");
assertThat(instance.getProcess(ScheduleActions.SCHEDULED).getIsHidden()).isTrue();
assertThat(instance.getProcess(ScheduleActions.SEND).getTableName()).isEqualTo(DataAnalysisTables.SCHEDULE);
var input = new RunBackendStepInput().withProcessName(ScheduleActions.SEND).withRecords(List.of(schedule));
var output = new RunBackendStepOutput();
var review = new ScheduleActions.Review();
review.runOnePage(input, output);
assertThat(review.getProcessSummary(output, false).toString()).contains("authorized recipients");
// Runtime has no messaging config: Result reports that real failure rather than claiming success.
var result = new ScheduleActions.Result();
result.runOnePage(input, new RunBackendStepOutput());
assertThat(result.getProcessSummary(output, true).toString()).contains("No messaging provider");
QContext.init(instance, SavedAnalysesTest.viewer("bob", 2));
assertThatThrownBy(() -> runner.sendNow(schedule.getValue("id"), NOW)).isInstanceOf(QException.class);
}
@Test
void recipientMustHaveFullFieldAccessAndHtmlFormatHasNoAttachment() throws Exception
{
var schedule = schedule("TABLE_PROFILE", Map.of("fields", List.of("amount", "status")), "HTML_SUMMARY");
config.withFieldAnalysisAccessResolver(() -> (table, field) ->
field.getName().equals("amount") && QContext.getQSession().getUser().getIdReference().equals("bob")
? com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccess.NONE : com.kingsrook.qbits.dataanalysis.engine.FieldAnalysisAccess.FULL);
var restricted = new ScheduleRunner(config, new DataAnalysisService(config));
assertThat(restricted.sendNow(schedule.getValue("id"), NOW)).contains("Sent 1", "bob", "denied", "missing");
assertThat(Mail.sent).hasSize(1);
assertThat(Mail.sent.getFirst().getAttachmentList()).isNullOrEmpty();
var output = new RunBackendStepOutput();
new ScheduleActions.Scheduled().run(new RunBackendStepInput(), output);
assertThat(output.getValue("count")).isNotNull();
}
@Test
void stockSmtpAttachmentLimitationFailsBeforeSending() throws Exception
{
var schedule = schedule("COLUMN_PROFILE", Map.of("fieldName", "amount"), "CSV");
var smtp = new com.kingsrook.qqq.backend.core.model.metadata.messaging.email.EmailMessagingProviderMetaData();
smtp.withName("smtp");
instance.addMessagingProvider(smtp);
config.withMessagingProviderName("smtp");
assertThat(runner.sendNow(schedule.getValue("id"), NOW)).contains("does not support attachments");
assertThat(Mail.sent).isEmpty();
assertThat(WatchesTest.Sink.raised).hasSize(1);
}
}
@@ -116,7 +116,7 @@ final class TestSupport
*******************************************************************************/
static QRecord saved(String owner, boolean shared)
{
return (new QRecord().withValue("name", "Profile").withValue("table_name", "orders").withValue("kind", "COLUMN_PROFILE")
.withValue("definition_json", "{}").withValue("owner_user_id", owner).withValue("shared", shared));
return (new QRecord().withValue("name", "Profile").withValue("table_name", DataAnalysisTables.RUN).withValue("kind", "COLUMN_PROFILE")
.withValue("definition_json", "{\"fieldName\":\"table_name\"}").withValue("owner_user_id", owner).withValue("shared", shared));
}
}
@@ -319,9 +319,9 @@ class WatchesTest
evaluator.setEnabled(disabled.getValue("id"), false);
assertThat(evaluator.evaluateEnabled(NOW)).isEqualTo(1);
assertThat(stored(disabled).getValue("last_evaluated_at")).isNull();
String html = DataAnalysisHome.render(NOW);
assertThat(html.indexOf("Firing &middot; Spike")).isLessThan(html.indexOf("Ok &middot; Drop"));
assertThat(html).contains("/" + spike.getValue("id") + "\"");
var data = (com.kingsrook.qqq.backend.core.model.dashboard.widgets.TableData) DataAnalysisHome.render(DataAnalysisHome.FIRING, NOW);
assertThat(data.getRows()).hasSize(1);
assertThat(data.getRows().getFirst().get("c0").toString()).contains("/" + spike.getValue("id") + "\"");
assertThat(instance.getProcess(WatchActions.SCHEDULED).getIsHidden()).isTrue();
assertThat(instance.getProcess(WatchActions.EVALUATE).getTableName()).isEqualTo(DataAnalysisTables.WATCH);
}
@@ -350,7 +350,7 @@ class WatchesTest
void savedAnalysisResolvesAsOwnerAndNoAlertsInstallationStillStoresState() throws Exception
{
QRecord saved = TestSupport.insert(DataAnalysisTables.SAVED, TestSupport.saved("alice", false)
.withValue("definition_json", "{\"values\":[{\"operator\":\"COUNT\"}]}"));
.withValue("kind", "PIVOT").withValue("table_name", "orders").withValue("definition_json", "{\"values\":[{\"operator\":\"COUNT\"}]}"));
QRecord definition = WatchDetectorsTest.definition("SPIKE", "COUNT", "{\"k\":3}")
.withValue("saved_id", saved.getValue("id")).withValue("table_name", null).withValue("metric_json", null);
QRecord watch = TestSupport.insert(DataAnalysisTables.WATCH, definition);