From d8b52113caed47ca5af6977ce0863b5004178fde Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 13 Oct 2026 18:26:45 -0500 Subject: [PATCH] feat(results): human labels, formatted values, nice bins, date labels, current home Every metric carries a plain-English label and a display-formatted value (grouping, percent, the field's display format), histogram bins use nice edges with formatted labels, date buckets are labelled by unit, warnings appear only when true, and the Analyses home shows saved analyses, watches and recent runs with honest empty states (and no mojibake). Co-Authored-By: Claude Opus 5.5 --- .../dataanalysis/api/DataAnalysisSchemas.java | 24 +- .../dataanalysis/app/DataAnalysisHome.java | 87 +++++-- .../dataanalysis/engine/AnalysisBackend.java | 13 +- .../engine/AnalysisFormatting.java | 225 ++++++++++++++++++ .../dataanalysis/engine/ColumnProfiler.java | 30 ++- .../engine/CoreAnalysisBackend.java | 20 +- .../dataanalysis/engine/MetricLabels.java | 65 +++++ .../engine/MinGroupSizeGuard.java | 4 +- .../qbits/dataanalysis/engine/NiceBins.java | 62 +++++ .../dataanalysis/engine/ProfileSupport.java | 11 +- .../engine/StandInAnalysisBackend.java | 46 +++- .../dataanalysis/engine/TableProfiler.java | 2 +- .../openapi/qqq-v1/data-analysis.yaml | 15 +- .../dataanalysis/AnalysisFormattingTest.java | 158 ++++++++++++ .../dataanalysis/CoreAnalysisBackendTest.java | 5 +- .../dataanalysis/DataAnalysisHomeTest.java | 45 +++- .../dataanalysis/EngineContractTest.java | 63 ++++- .../qbits/dataanalysis/EngineFixture.java | 3 +- .../PresentationSamplingTest.java | 115 +++++++++ 19 files changed, 901 insertions(+), 92 deletions(-) create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java create mode 100644 src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java create mode 100644 src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java index 794dbd5..151c72e 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java @@ -603,13 +603,13 @@ public class DataAnalysisSchemas @OpenAPIDescription("Type.") public String type; - @OpenAPIDescription("Completeness.") + @OpenAPIDescription("Non-empty share as a 0..1 ratio rounded to six decimals.") public BigDecimal completeness; @OpenAPIDescription("Distinct.") public Long distinct; - @OpenAPIDescription("Uniqueness.") + @OpenAPIDescription("Distinct share as a 0..1 ratio rounded to six decimals.") public BigDecimal uniqueness; @OpenAPIDescription("Constant.") @@ -634,9 +634,9 @@ public class DataAnalysisSchemas properties.put("field", new Schema().withType(Type.STRING).withDescription("Field.").withNullable(true)); properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); properties.put("type", new Schema().withType(Type.STRING).withDescription("Type.").withNullable(true)); - properties.put("completeness", new Schema().withType(Type.NUMBER).withDescription("Completeness.").withNullable(true)); + properties.put("completeness", new Schema().withType(Type.NUMBER).withDescription("Non-empty share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("distinct", new Schema().withType(Type.INTEGER).withDescription("Distinct.").withNullable(true)); - properties.put("uniqueness", new Schema().withType(Type.NUMBER).withDescription("Uniqueness.").withNullable(true)); + properties.put("uniqueness", new Schema().withType(Type.NUMBER).withDescription("Distinct share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("constant", new Schema().withType(Type.BOOLEAN).withDescription("Constant.").withNullable(true)); properties.put("empty", new Schema().withType(Type.BOOLEAN).withDescription("Empty.").withNullable(true)); properties.put("nulls", ref("DataAnalysisMetricV1").withDescription("Nulls.").withNullable(true)); @@ -654,16 +654,16 @@ public class DataAnalysisSchemas public Boolean estimate; public String key; - @OpenAPIDescription("Label.") + @OpenAPIDescription("Plain-English metric label, ready to display.") public String label; - @OpenAPIDescription("Value.") + @OpenAPIDescription("Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded to six decimals; DECIMAL field statistics have ten significant digits. DATE_TIME is an ISO instant.") public Serializable value; @OpenAPIDescription("Value type.") public ValueType valueType; - @OpenAPIDescription("Formatted.") + @OpenAPIDescription("Display-ready value, including grouping, field display formats, percent signs and session-zone dates; null when absent or suppressed.") public String formatted; @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") @@ -683,11 +683,11 @@ public class DataAnalysisSchemas { Map properties = new LinkedHashMap<>(); properties.put("key", new Schema().withType(Type.STRING).withDescription("Key.").withNullable(true)); - properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); - properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Value.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Plain-English metric label, ready to display.").withNullable(true)); + properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded to six decimals; DECIMAL field statistics have ten significant digits. DATE_TIME is an ISO instant.").withNullable(true)); properties.put("valueType", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ValueType.values()).map(Enum::name).toList()).withDescription("Value type.").withNullable(true)); properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("Whether this metric is estimated; the population rows metric remains exact.")); - properties.put("formatted", new Schema().withType(Type.STRING).withDescription("Formatted.").withNullable(true)); + properties.put("formatted", new Schema().withType(Type.STRING).withDescription("Display-ready value, including grouping, field display formats, percent signs and session-zone dates; null when absent or suppressed.").withNullable(true)); properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); properties.put("suppressed", new Schema().withType(Type.BOOLEAN).withDescription("Suppressed.").withNullable(true)); properties.put("suppressedReason", new Schema().withType(Type.STRING).withDescription("Suppressed reason.").withNullable(true)); @@ -961,7 +961,7 @@ public class DataAnalysisSchemas @OpenAPIDescription("Count.") public Long count; - @OpenAPIDescription("Share.") + @OpenAPIDescription("Share as a 0..1 ratio rounded to six decimals.") public BigDecimal share; @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") @@ -977,7 +977,7 @@ public class DataAnalysisSchemas properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Value.").withNullable(true)); properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); properties.put("count", new Schema().withType(Type.INTEGER).withDescription("Count.").withNullable(true)); - properties.put("share", new Schema().withType(Type.NUMBER).withDescription("Share.").withNullable(true)); + properties.put("share", new Schema().withType(Type.NUMBER).withDescription("Share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java index b8d0bb8..fa345a9 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java @@ -22,26 +22,33 @@ package com.kingsrook.qbits.dataanalysis.app; import java.time.Instant; -import java.time.temporal.ChronoUnit; +import java.util.Arrays; import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting; import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; @@ -93,7 +100,8 @@ public class DataAnalysisHome extends AbstractWidgetRenderer /******************************************************************************* - ** Standard QQQ widget renderer entry point. + ** RawHTML is carried as a JSON string by the HTML widget, not a text/html response. + ** Entities keep static separators independent of any host charset handling. *******************************************************************************/ @Override public RenderWidgetOutput render(RenderWidgetInput input) throws QException @@ -110,7 +118,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { StringBuilder html = new StringBuilder("
"); String userId = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? null : QContext.getQSession().getUser().getIdReference(); - html.append("

Saved analyses

"); + html.append("

Saved analyses

"); if(userId == null) { html.append("

Sign in to see your saved analyses and stars.

"); @@ -119,7 +127,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { int mine = count(DataAnalysisTables.SAVED, equal("owner_user_id", userId)); int shared = count(DataAnalysisTables.SAVED, equal("shared", true)); - html.append("

Mine: ").append(mine).append(" · Shared (visible to you): ").append(shared).append("

"); + html.append("

Mine: ").append(AnalysisFormatting.number(mine, 0)).append(" · Shared (visible to you): ").append(AnalysisFormatting.number(shared, 0)).append("

"); if(canRead(DataAnalysisTables.SAVED_STAR)) { CountInput stars = new CountInput(DataAnalysisTables.SAVED).withIncludeDistinctCount(true) @@ -127,28 +135,28 @@ public class DataAnalysisHome extends AbstractWidgetRenderer .withFilter(equal(DataAnalysisTables.SAVED_STAR + ".user_id", userId)); stars.setPermissionEnforcement(PermissionEnforcement.ENFORCE); int starred = new CountAction().execute(stars).getDistinctCount(); - html.append("

Starred by me: ").append(starred).append("

"); + html.append("

Starred by me: ").append(AnalysisFormatting.number(starred, 0)).append("

"); } if(mine == 0 && shared == 0) { - html.append("

No saved analyses yet. Saving analyses from the Analyze action is planned for a later release.

"); + html.append("

No saved analyses yet. Open any table and choose Analyze on a column header to explore its data.

"); } - html.append("

The Analyze action will be available in a later UI release.

"); link(html, DataAnalysisTables.SAVED); } else { html.append("

Saved analyses are unavailable: you need data_analysis.saved.read.

"); } - html.append("

Watches

"); + html.append("

Watches

"); if(canRead(DataAnalysisTables.WATCH)) { + int total = count(DataAnalysisTables.WATCH, new QQueryFilter()); int enabled = count(DataAnalysisTables.WATCH, equal("enabled", true)); int firing = count(DataAnalysisTables.WATCH, equal("state", "FIRING")); - html.append("

Enabled: ").append(enabled).append(" · Firing: ").append(firing).append("

"); - if(enabled == 0 && firing == 0) + html.append("

Total: ").append(AnalysisFormatting.number(total, 0)).append(" · Enabled: ").append(AnalysisFormatting.number(enabled, 0)).append(" · Firing: ").append(AnalysisFormatting.number(firing, 0)).append("

"); + if(total == 0) { - html.append("

No active watches: watch evaluation is planned for phase e.

"); + html.append("

No watches yet.

"); } link(html, DataAnalysisTables.WATCH); } @@ -156,15 +164,37 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { html.append("

Watches are unavailable: you need data_analysis.watch.read.

"); } - html.append("

Runs in the last 7 days

"); - if(canRead(DataAnalysisTables.RUN)) + html.append("

Recent analysis runs

"); + if(userId == null) { - int complete = count(DataAnalysisTables.RUN, recentRuns("COMPLETE", now)); - int failed = count(DataAnalysisTables.RUN, recentRuns("FAILED", now)); - html.append("

Complete: ").append(complete).append(" · Failed: ").append(failed).append("

"); - if(complete == 0 && failed == 0) + html.append("

Sign in to see your recent analysis runs.

"); + } + else if(canRead(DataAnalysisTables.RUN)) + { + QueryInput query = new QueryInput(DataAnalysisTables.RUN).withFilter(equal("user_id", userId) + .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)) + .withOrderBy(new QFilterOrderBy("started_at", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(5)); + query.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + List runs = new QueryAction().execute(query).getRecords(); + if(runs.isEmpty()) { - html.append("

No completed or failed runs in this period.

"); + html.append("

No analysis runs yet.

"); + } + else + { + html.append("
    "); + for(QRecord run : runs) + { + QTableMetaData table = QContext.getQInstance().getTable(run.getValueString("table_name")); + String tableLabel = table == null ? run.getValueString("table_name") : table.getLabel(); + String path = QContext.getQInstance().getTablePath(DataAnalysisTables.RUN) + "/" + run.getValueInteger("id"); + html.append("
  • ").append(escape(tableLabel)).append(" · ") + .append(escape(words(run.getValueString("operation")))).append("
    Status
    ") + .append(escape(words(run.getValueString("status")))).append("
    Finished
    ") + .append(run.getValue("finished_at") == null ? "Not finished" : escape(AnalysisFormatting.instant(run.getValueInstant("finished_at")))) + .append("
  • "); + } + html.append("
"); } link(html, DataAnalysisTables.RUN); } @@ -198,13 +228,22 @@ public class DataAnalysisHome extends AbstractWidgetRenderer /******************************************************************************* - ** Runs started within the inclusive seven-day interval, excluding future rows. + ** Stored operation and status keys are displayed as ordinary words. *******************************************************************************/ - private static QQueryFilter recentRuns(String status, Instant now) + private static String words(String value) { - return (equal("status", status) - .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.GREATER_THAN_OR_EQUALS, now.minus(7, ChronoUnit.DAYS))) - .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now))); + return (value == null ? "Unknown" : Arrays.stream(value.toLowerCase(Locale.ROOT).split("_")) + .filter(part -> !part.isEmpty()).map(part -> Character.toUpperCase(part.charAt(0)) + part.substring(1)).collect(Collectors.joining(" "))); + } + + + + /******************************************************************************* + ** Metadata and persisted values must never become executable widget markup. + *******************************************************************************/ + private static String escape(String value) + { + return (value == null ? "Unknown" : value.replace("&", "&").replace("\"", """).replace("<", "<").replace(">", ">").replace("'", "'")); } @@ -227,7 +266,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer private static void link(StringBuilder html, String table) throws QException { String path = QContext.getQInstance().getTablePath(table); - html.append("", ">")) + html.append("View all"); } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java index 14ebe27..77f122c 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java @@ -38,7 +38,7 @@ public interface AnalysisBackend *******************************************************************************/ enum DateUnit { - DAY, WEEK, MONTH, YEAR, DAY_OF_WEEK, HOUR_OF_DAY + DAY, WEEK, MONTH, QUARTER, YEAR, HOUR, DAY_OF_WEEK, HOUR_OF_DAY } @@ -120,6 +120,15 @@ public interface AnalysisBackend + /******************************************************************************* + ** Actual inspected non-null values, independent of estimated population counts. + *******************************************************************************/ + record PatternSummary(List groups, long values, boolean truncated) + { + } + + + /******************************************************************************* ** Filtered count; a native estimate scales conditional counts to its population. *******************************************************************************/ @@ -168,7 +177,7 @@ public interface AnalysisBackend /******************************************************************************* ** Top shapes over a bounded primary-key prefix. *******************************************************************************/ - List patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException; + PatternSummary patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException; /******************************************************************************* ** Top groups with count greater than one. diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java new file mode 100644 index 0000000..badae57 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java @@ -0,0 +1,225 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalDate; +import java.time.Period; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.TextStyle; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + +/******************************************************************************* + ** Display-ready values shared by metrics, chart labels and the home widget. + *******************************************************************************/ +public final class AnalysisFormatting +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private AnalysisFormatting() + { + } + + + + /******************************************************************************* + ** Ratios remain numeric and bounded, with stable JSON precision. + *******************************************************************************/ + public static BigDecimal ratio(BigDecimal value) + { + return (value.max(BigDecimal.ZERO).min(BigDecimal.ONE).setScale(6, RoundingMode.HALF_UP).stripTrailingZeros()); + } + + + + /******************************************************************************* + ** Plain numeric defaults are independent of the server locale. + *******************************************************************************/ + public static String number(Number value, int decimals) + { + return (String.format(Locale.US, "%,." + decimals + "f", new BigDecimal(value.toString()))); + } + + + + /******************************************************************************* + ** Types and length keys determine display without changing numeric raw values. + *******************************************************************************/ + public static String scalar(String key, Serializable value, ValueType type) + { + if(value == null) + { + return (null); + } + if(type == ValueType.PERCENT) + { + BigDecimal percent = new BigDecimal(value.toString()).multiply(BigDecimal.valueOf(100)).setScale(1, RoundingMode.HALF_UP); + return (percent.signum() == 0 || percent.compareTo(BigDecimal.valueOf(100)) == 0 ? percent.toBigInteger() + "%" : percent.toPlainString() + "%"); + } + if(type == ValueType.DATE_TIME) + { + return (instant((Instant) value)); + } + if(value instanceof Number n) + { + return (number(n, "meanLength".equals(key) ? 1 : type == ValueType.COUNT || key.endsWith("Length") ? 0 : 2)); + } + return (value.toString()); + } + + + + /******************************************************************************* + ** Display formats belong to the host field; fallback defaults preserve fractions. + *******************************************************************************/ + public static String numeric(QFieldMetaData field, String key, BigDecimal value) + { + if(value == null) + { + return (null); + } + String format = field.getDisplayFormat(); + if(format != null && !format.isBlank() && !DisplayFormat.DEFAULT.equals(format)) + { + Serializable displayValue = format.endsWith("d") ? value.toBigInteger() : value; + return (QValueFormatter.formatValue(format, displayValue)); + } + return (number(value, field.getType() == QFieldType.DECIMAL || "mean".equals(key) || "stddev".equals(key) ? 2 : 0)); + } + + + + /******************************************************************************* + ** Decimal statistics retain ten significant digits; formatting uses the original. + *******************************************************************************/ + public static Metric numericMetric(QFieldMetaData field, String key, BigDecimal value) + { + BigDecimal raw = value == null || field.getType() != QFieldType.DECIMAL ? value : value.round(new MathContext(10)); + return (ProfileSupport.metric(key, raw, ValueType.DECIMAL).withFormatted(numeric(field, key, value))); + } + + + + /******************************************************************************* + ** Use the same field formatter on both ends of every histogram interval. + *******************************************************************************/ + public static String interval(QFieldMetaData field, BigDecimal low, BigDecimal high) + { + return (low == null ? "Below " + edge(field, high) + : high == null ? edge(field, low) + " and above" + : edge(field, low) + " – " + edge(field, high)); + } + + + + /******************************************************************************* + ** Whole edges omit redundant cents; unformatted fractional edges stay distinct. + *******************************************************************************/ + private static String edge(QFieldMetaData field, BigDecimal value) + { + String format = field.getDisplayFormat(); + int scale = Math.max(0, value.stripTrailingZeros().scale()); + if(format == null || format.isBlank() || DisplayFormat.DEFAULT.equals(format)) + { + return (number(value, scale)); + } + String formatted = numeric(field, "min", value); + return (scale == 0 && DisplayFormat.CURRENCY.equals(format) && formatted.endsWith(".00") + ? formatted.substring(0, formatted.length() - 3) : formatted); + } + + + + /******************************************************************************* + ** Instants remain UTC in raw values and display in the current session zone. + *******************************************************************************/ + public static String instant(Instant value) + { + return (DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm", Locale.US).format(value.atZone(ValueUtils.getSessionOrInstanceZoneId()))); + } + + + + /******************************************************************************* + ** Calendar duration avoids misleading fixed-length month and year arithmetic. + *******************************************************************************/ + public static String duration(LocalDate first, LocalDate last) + { + Period period = Period.between(first, last); + List parts = new ArrayList<>(); + if(period.getYears() != 0) + { + parts.add(period.getYears() + (period.getYears() == 1 ? " year" : " years")); + } + if(period.getMonths() != 0) + { + parts.add(period.getMonths() + (period.getMonths() == 1 ? " month" : " months")); + } + if(period.getDays() != 0 || parts.isEmpty()) + { + parts.add(period.getDays() + (period.getDays() == 1 ? " day" : " days")); + } + return (String.join(" ", parts)); + } + + + + /******************************************************************************* + ** Calendar labels describe one bucket; the raw value remains its ISO start. + *******************************************************************************/ + public static String dateBucket(Serializable start, AnalysisBackend.DateUnit unit, ZoneId zone) + { + if(unit == AnalysisBackend.DateUnit.DAY_OF_WEEK) + { + return (DayOfWeek.of(((Number) start).intValue()).getDisplayName(TextStyle.FULL, Locale.US)); + } + if(unit == AnalysisBackend.DateUnit.HOUR_OF_DAY) + { + return (String.format(Locale.US, "%02d:00", ((Number) start).intValue())); + } + LocalDate date = ProfileSupport.day(start, zone); + return (switch(unit) + { + case YEAR -> Integer.toString(date.getYear()); + case QUARTER -> date.getYear() + " Q" + ((date.getMonthValue() - 1) / 3 + 1); + case MONTH -> DateTimeFormatter.ofPattern("MMM uuuu", Locale.US).format(date); + case WEEK -> "Week of " + date; + case HOUR -> DateTimeFormatter.ofPattern("uuuu-MM-dd HH:00", Locale.US).format(((Instant) start).atZone(zone)); + default -> date.toString(); + }); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java index 55edeb3..27945b2 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java @@ -86,7 +86,7 @@ public class ColumnProfiler QFieldType type = field.getType(); if(type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.DECIMAL) { - numeric(result, filter, drillFilter, rows); + numeric(result, filter, drillFilter, rows, field); } else if(type == QFieldType.DATE || type == QFieldType.DATE_TIME) { @@ -95,13 +95,18 @@ public class ColumnProfiler else if(type.isStringLike()) { AnalysisBackend.LengthSummary summary = backend.lengthSummary(table, filter, name); + AnalysisBackend.PatternSummary patterns = backend.patterns(table, filter, name, 10_000); + long patternValues = patterns.values(); TextProfile text = new TextProfile().withHistogram(ProfileSupport.values(table, drillFilter, null, summary.histogram(), rows)) - .withPatterns(ProfileSupport.values(table, drillFilter, null, backend.patterns(table, filter, name, 10_000), Math.min(rows, 10_000))) - .withSampleRows(Math.min(scope.rowsScanned(), 10_000)); + .withPatterns(ProfileSupport.values(table, drillFilter, null, patterns.groups(), patternValues)) + .withSampleRows(patternValues); summary.metrics().forEach((key, value) -> text.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); result.withText(text); - result.getWarnings().add(scope.samplePercent() == null ? "Patterns use the first " + Math.min(rows, 10_000) + " rows by primary key." - : "Patterns use up to " + Math.min(scope.rowsScanned(), 10_000) + " sampled rows."); + if(patterns.truncated()) + { + result.getWarnings().add(scope.samplePercent() == null ? "Patterns are based on the first " + AnalysisFormatting.number(patternValues, 0) + " values." + : "Patterns are based on " + AnalysisFormatting.number(patternValues, 0) + " sampled values."); + } } else if(type == QFieldType.BOOLEAN) { @@ -142,14 +147,15 @@ public class ColumnProfiler else { result.withChart(ProfileSupport.chart(ChartKind.TIME_SERIES, ChartKind.BAR, name, result.getDates().getBuckets())); + result.getChart().getX().withLabel(field.getLabel() + (result.getDates().getUnit() == null ? "" : " (" + result.getDates().getUnit().toLowerCase(java.util.Locale.ROOT) + ")")); } - if(rows == 0) + if(scope.total() == 0) { result.getWarnings().add("No readable records match this analysis scope."); } - else if(counts.nonNull() == 0 && (!scope.estimate() || scope.rowsScanned() > 0)) + else if(counts.nonNull() == 0 && rows > 0 && (!scope.estimate() || scope.rowsScanned() > 0)) { - result.getWarnings().add("All values in this analysis scope are null."); + result.getWarnings().add(scope.estimate() ? "All observed values are null." : "All values in this analysis scope are null."); } if(scope.estimate()) { @@ -170,16 +176,16 @@ public class ColumnProfiler /******************************************************************************* - ** Numeric moments, twenty interior bins, exterior bins and Tukey outliers. + ** Numeric moments, nice interior bins, exterior bins and Tukey outliers. *******************************************************************************/ - private void numeric(ColumnProfile result, QQueryFilter filter, QQueryFilter drillFilter, long rows) throws QException + private void numeric(ColumnProfile result, QQueryFilter filter, QQueryFilter drillFilter, long rows, QFieldMetaData metadata) throws QException { String table = result.getTableName(); String field = result.getFieldName(); AnalysisBackend.NumericSummary summary = backend.numericSummary(table, filter, field, List.of("0.01", "0.05", "0.25", "0.50", "0.75", "0.95", "0.99").stream().map(BigDecimal::new).toList()); NumericProfile numeric = new NumericProfile(); - summary.metrics().forEach((key, value) -> numeric.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); + summary.metrics().forEach((key, value) -> numeric.getMetrics().add(AnalysisFormatting.numericMetric(metadata, key, value))); for(AnalysisBackend.GroupCount count : summary.counts()) { numeric.getMetrics().add(ProfileSupport.count(count.label(), count.count(), table, drillFilter, count.condition())); @@ -211,7 +217,7 @@ public class ColumnProfiler LocalDate last = ProfileSupport.day(extrema.get("max"), zone); AnalysisBackend.DateUnit unit = autoBucket(first, last); dates.withUnit(unit.name()); - dates.getMetrics().add(ProfileSupport.metric("range", ChronoUnit.DAYS.between(first, last), ValueType.DURATION)); + dates.getMetrics().add(ProfileSupport.metric("range", ChronoUnit.DAYS.between(first, last), ValueType.DURATION).withFormatted(AnalysisFormatting.duration(first, last))); List buckets = ProfileSupport.values(table, drillFilter, null, backend.dateBuckets(table, filter, field, unit, zone), rows); dates.withBuckets(buckets).withGaps(buckets.stream().filter(b -> b.getCount() == 0).toList()); if(type == QFieldType.DATE_TIME) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java index e5f9201..3520581 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java @@ -85,7 +85,7 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend { samplePercent = null; long total = super.count(table, filter); - if(exact || total <= config.getSampleThresholdRows() || !capabilities(table).contains(AggregateCapability.SAMPLING)) + if(exact || total <= config.getSampleThresholdRows() || total <= config.getSampleRows() || !capabilities(table).contains(AggregateCapability.SAMPLING)) { return (super.scope(table, filter, exact)); } @@ -375,21 +375,25 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend { return (super.numericHistogram(table, filter, field, low, high, bins)); } + NiceBins nice = NiceBins.covering(low, high); + low = nice.low(); + high = nice.high(); + bins = nice.count(); GroupBy group = new GroupBy(field(table, field)).withBucket(new NumericBucket(low, high, bins)); Map counts = bucketCounts(table, filter, group); List values = !estimating && isMemory(table) ? rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList() : List.of(); List result = new ArrayList<>(); - result.add(new GroupCount("Below " + low, "Below " + low, aggregateCount(counts.getOrDefault(0, 0L)), numericRange(table, field, values, null, low, false), aggregateShare(counts.getOrDefault(0, 0L)))); + result.add(new GroupCount(AnalysisFormatting.interval(field(table, field), null, low), AnalysisFormatting.interval(field(table, field), null, low), aggregateCount(counts.getOrDefault(0, 0L)), numericRange(table, field, values, null, low, false), aggregateShare(counts.getOrDefault(0, 0L)))); BigDecimal width = high.subtract(low).divide(BigDecimal.valueOf(bins), MathContext.DECIMAL128); for(int i = 0; i < bins; i++) { BigDecimal start = low.add(width.multiply(BigDecimal.valueOf(i))); BigDecimal end = i == bins - 1 ? high : low.add(width.multiply(BigDecimal.valueOf(i + 1))); - String label = "[" + start + ", " + end + (i == bins - 1 ? "]" : ")"); + String label = AnalysisFormatting.interval(field(table, field), start, end); result.add(new GroupCount(label, label, aggregateCount(counts.getOrDefault(i + 1, 0L)), numericRange(table, field, values, start, end, i == bins - 1), aggregateShare(counts.getOrDefault(i + 1, 0L)))); } - result.add(new GroupCount("Above " + high, "Above " + high, aggregateCount(counts.getOrDefault(bins + 1, 0L)), + result.add(new GroupCount(AnalysisFormatting.interval(field(table, field), high, null), AnalysisFormatting.interval(field(table, field), high, null), aggregateCount(counts.getOrDefault(bins + 1, 0L)), DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)), aggregateShare(counts.getOrDefault(bins + 1, 0L)))); return (result); } @@ -400,6 +404,10 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend @Override public List dateBuckets(String table, QQueryFilter filter, String field, DateUnit unit, ZoneId zone) throws QException { + if(unit == DateUnit.HOUR || unit == DateUnit.QUARTER) + { + return (super.dateBuckets(table, filter, field, unit, zone)); + } GroupBy group = new GroupBy(field(table, field)).withBucket(new DateBucket(DateBucketUnit.valueOf(unit.name()))); AggregateInput probe = new AggregateInput(table).withGroupBy(group); if(!new QBackendModuleDispatcher().getQBackendModule(probe.getBackend()).getAggregateInterface().getSupportedCapabilities(probe).contains(AggregateCapability.DATE_BUCKET)) @@ -423,8 +431,8 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend while(!start.isAfter(dates.getLast())) { LocalDate end = next(start, unit); - String label = start + " / " + end; - result.add(new GroupCount(label, label, aggregateCount(counts.getOrDefault(start, 0L)), + String label = AnalysisFormatting.dateBucket(start, unit, zone); + result.add(new GroupCount(start, label, aggregateCount(counts.getOrDefault(start, 0L)), DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone), aggregateShare(counts.getOrDefault(start, 0L)))); start = end; } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java new file mode 100644 index 0000000..2b7ce09 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qbits.dataanalysis.result.ValueType; + +/******************************************************************************* + ** One vocabulary for every emitted metric, including date-specific extrema. + *******************************************************************************/ +public final class MetricLabels +{ + private static final Map LABELS = Map.ofEntries( + Map.entry("rows", "Rows"), Map.entry("nonNull", "Non-empty"), Map.entry("nulls", "Nulls"), + Map.entry("blanks", "Blanks"), Map.entry("distinct", "Distinct values"), Map.entry("uniqueness", "Uniqueness"), + Map.entry("completeness", "Completeness"), Map.entry("share", "Share"), + Map.entry("min", "Minimum"), Map.entry("max", "Maximum"), Map.entry("mean", "Mean"), + Map.entry("median", "Median"), Map.entry("stddev", "Standard deviation"), Map.entry("sum", "Sum"), + Map.entry("zeros", "Zeros"), Map.entry("negatives", "Negatives"), Map.entry("outliers", "Outliers"), + Map.entry("p1", "1st percentile"), Map.entry("p5", "5th percentile"), Map.entry("p25", "25th percentile (Q1)"), + Map.entry("p50", "Median"), Map.entry("p75", "75th percentile (Q3)"), Map.entry("p95", "95th percentile"), Map.entry("p99", "99th percentile"), + Map.entry("minLength", "Shortest"), Map.entry("maxLength", "Longest"), Map.entry("meanLength", "Average length"), + Map.entry("earliest", "Earliest"), Map.entry("latest", "Latest"), Map.entry("range", "Range"), + Map.entry("true", "True"), Map.entry("false", "False"), Map.entry("other", "Other")); + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private MetricLabels() + { + } + + + + /******************************************************************************* + ** Fail at construction when a new metric lacks a human label. + *******************************************************************************/ + public static String label(String key, ValueType type) + { + if(type == ValueType.DATE || type == ValueType.DATE_TIME) + { + key = "min".equals(key) ? "earliest" : "max".equals(key) ? "latest" : key; + } + return (Objects.requireNonNull(LABELS.get(key), "Missing metric label: " + key)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java index c9a82a2..3331d96 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java @@ -71,7 +71,7 @@ public class MinGroupSizeGuard } if(hidden > 0) { - result.add(new ValueCount().withLabel("Other (" + hidden + " groups below " + minimum + ")").withCount(count).withShare(share)); + result.add(new ValueCount().withLabel("Other (" + hidden + " groups below " + minimum + ")").withCount(count).withShare(AnalysisFormatting.ratio(share))); } return (result); } @@ -114,7 +114,7 @@ public class MinGroupSizeGuard List booleanGroups = profile.getBooleans().stream().map(m -> new ValueCount().withValue(m.getKey()).withLabel(m.getLabel()) .withCount(((Number) m.getValue()).longValue()).withShare(ProfileSupport.share(((Number) m.getValue()).longValue(), rows)).withDrill(m.getDrill())).toList(); profile.withBooleans(groups(booleanGroups).stream().map(v -> ProfileSupport.metric(v.getValue() == null ? "other" : v.getValue().toString(), v.getCount(), - com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withLabel(v.getLabel()).withDrill(v.getDrill())).toList()); + com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withDrill(v.getDrill())).toList()); } } if(profile.getNumeric() != null) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java new file mode 100644 index 0000000..76807f3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.List; + +/******************************************************************************* + ** Exact decimal boundaries with a readable step and approximately twenty bins. + *******************************************************************************/ +public record NiceBins(BigDecimal low, BigDecimal high, int count) +{ + /******************************************************************************* + ** Select the closest 10..25-bin enclosure from 1, 2, 2.5 and 5 times powers of ten. + *******************************************************************************/ + public static NiceBins covering(BigDecimal low, BigDecimal high) + { + BigDecimal span = high.subtract(low); + if(span.signum() == 0) + { + span = low.abs().max(BigDecimal.ONE); + low = low.subtract(span.divide(BigDecimal.valueOf(2))); + high = high.add(span.divide(BigDecimal.valueOf(2))); + } + int exponent = span.precision() - span.scale() - 2; + NiceBins best = null; + for(int power = exponent - 2; power <= exponent + 2; power++) + { + for(String factor : List.of("1", "2", "2.5", "5")) + { + BigDecimal step = new BigDecimal(factor).scaleByPowerOfTen(power); + BigDecimal first = low.divide(step, 0, RoundingMode.FLOOR).multiply(step); + BigDecimal last = high.divide(step, 0, RoundingMode.CEILING).multiply(step); + int count = last.subtract(first).divide(step).intValueExact(); + if(count >= 10 && count <= 25 && (best == null || Math.abs(count - 20) < Math.abs(best.count() - 20))) + { + best = new NiceBins(first, last, count); + } + } + } + return (java.util.Objects.requireNonNull(best)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java index 83c8b59..716de31 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java @@ -56,7 +56,7 @@ public final class ProfileSupport *******************************************************************************/ public static BigDecimal share(long count, long rows) { - return (rows == 0 ? BigDecimal.ZERO : BigDecimal.valueOf(count).divide(BigDecimal.valueOf(rows), MathContext.DECIMAL128)); + return (rows == 0 ? BigDecimal.ZERO : BigDecimal.valueOf(count).divide(BigDecimal.valueOf(rows), 6, java.math.RoundingMode.HALF_UP).stripTrailingZeros()); } @@ -76,7 +76,12 @@ public final class ProfileSupport *******************************************************************************/ public static Metric metric(String key, Serializable value, ValueType type) { - return (new Metric().withKey(key).withLabel(key).withValue(value).withValueType(type).withFormatted(value == null ? null : value.toString())); + if(type == ValueType.PERCENT && value != null) + { + value = AnalysisFormatting.ratio(new BigDecimal(value.toString())); + } + return (new Metric().withKey(key).withLabel(MetricLabels.label(key, type)).withValue(value).withValueType(type) + .withFormatted(AnalysisFormatting.scalar(key, value, type))); } @@ -116,7 +121,7 @@ public final class ProfileSupport label = translated; } } - result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(group.share() == null ? share(group.count(), total) : group.share()) + result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(group.share() == null ? share(group.count(), total) : AnalysisFormatting.ratio(group.share())) .withDrill(DrillFilters.drill(table, filter, group.condition()))); } return (result); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java index a8ee186..48b342f 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java @@ -101,7 +101,7 @@ public class StandInAnalysisBackend implements AnalysisBackend public Scope scope(String table, QQueryFilter filter, boolean exact) throws QException { long total = count(table, filter); - boolean estimate = !exact && total > config.getSampleThresholdRows(); + boolean estimate = !exact && total > config.getSampleThresholdRows() && total > Math.min(config.getSampleRows(), config.getMaxRows()); estimating = estimate; int limit = estimate ? Math.min(config.getSampleRows(), config.getMaxRows()) : config.getMaxRows(); QQueryFilter scoped = DrillFilters.copy(filter); @@ -455,17 +455,21 @@ public class StandInAnalysisBackend implements AnalysisBackend { return (result); } + NiceBins nice = NiceBins.covering(low, high); + low = nice.low(); + high = nice.high(); + bins = nice.count(); List values = rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList(); - result.add(bucket(table, filter, "Below " + low, numericRange(table, field, values, null, low, false))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), null, low), numericRange(table, field, values, null, low, false))); BigDecimal width = high.subtract(low).divide(BigDecimal.valueOf(bins), MATH); for(int i = 0; i < bins; i++) { BigDecimal start = low.add(width.multiply(BigDecimal.valueOf(i))); BigDecimal end = i == bins - 1 ? high : low.add(width.multiply(BigDecimal.valueOf(i + 1))); - result.add(bucket(table, filter, "[" + start + ", " + end + (i == bins - 1 ? "]" : ")"), numericRange(table, field, values, start, end, i == bins - 1))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), start, end), numericRange(table, field, values, start, end, i == bins - 1))); } - result.add(bucket(table, filter, "Above " + high, DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), high, null), DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)))); return (result); } @@ -491,7 +495,26 @@ public class StandInAnalysisBackend implements AnalysisBackend if(unit == DateUnit.DAY_OF_WEEK || unit == DateUnit.HOUR_OF_DAY) { return (mappedGroups(table, filter, field, records, v -> unit == DateUnit.DAY_OF_WEEK ? day(v, zone).getDayOfWeek().getValue() - : ((Instant) v).atZone(zone).getHour(), 24)); + : ((Instant) v).atZone(zone).getHour(), 24).stream() + .map(g -> new GroupCount(g.value(), AnalysisFormatting.dateBucket(g.value(), unit, zone), g.count(), g.condition(), g.share())).toList()); + } + if(unit == DateUnit.HOUR) + { + List instants = records.stream().map(r -> (Instant) r.getValue(field)).filter(Objects::nonNull).sorted().toList(); + List hours = new ArrayList<>(); + if(!instants.isEmpty()) + { + Instant start = instants.getFirst().atZone(zone).truncatedTo(java.time.temporal.ChronoUnit.HOURS).toInstant(); + while(!start.isAfter(instants.getLast())) + { + Instant end = start.plus(1, java.time.temporal.ChronoUnit.HOURS); + QQueryFilter condition = DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN_OR_EQUALS, start), + DrillFilters.condition(field, QCriteriaOperator.LESS_THAN, end)); + hours.add(new GroupCount(start, AnalysisFormatting.dateBucket(start, unit, zone), count(table, DrillFilters.and(filter, condition)), condition, countShare())); + start = end; + } + } + return (hours); } List dates = records.stream().map(r -> r.getValue(field)).filter(Objects::nonNull).map(v -> day(v, zone)).sorted().toList(); List result = new ArrayList<>(); @@ -504,7 +527,8 @@ public class StandInAnalysisBackend implements AnalysisBackend while(!start.isAfter(last)) { LocalDate end = next(start, unit); - result.add(bucket(table, filter, start + " / " + end, DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone))); + QQueryFilter condition = DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone); + result.add(new GroupCount(start, AnalysisFormatting.dateBucket(start, unit, zone), count(table, DrillFilters.and(filter, condition)), condition, countShare())); start = end; } return (result); @@ -531,6 +555,7 @@ public class StandInAnalysisBackend implements AnalysisBackend { case WEEK -> date.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)); case MONTH -> date.withDayOfMonth(1); + case QUARTER -> date.withMonth((date.getMonthValue() - 1) / 3 * 3 + 1).withDayOfMonth(1); case YEAR -> date.withDayOfYear(1); default -> date; }); @@ -547,6 +572,7 @@ public class StandInAnalysisBackend implements AnalysisBackend { case WEEK -> date.plusWeeks(1); case MONTH -> date.plusMonths(1); + case QUARTER -> date.plusMonths(3); case YEAR -> date.plusYears(1); default -> date.plusDays(1); }); @@ -605,12 +631,14 @@ public class StandInAnalysisBackend implements AnalysisBackend ** Shapes use a primary-key prefix of at most the requested sample size. *******************************************************************************/ @Override - public List patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException + public PatternSummary patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException { - List records = rows(table, filter, Set.of(field, primaryKey(table)), sampleRows); + List records = rows(table, DrillFilters.and(filter, DrillFilters.nonNull(field, true)), Set.of(field, primaryKey(table)), sampleRows + 1); + boolean truncated = records.size() > sampleRows; + records = records.subList(0, Math.min(records.size(), sampleRows)); QQueryFilter sampled = records.isEmpty() ? DrillFilters.in(primaryKey(table), List.of()) : DrillFilters.condition(primaryKey(table), QCriteriaOperator.LESS_THAN_OR_EQUALS, records.getLast().getValue(primaryKey(table))); - return (mappedGroups(table, estimating ? filter : DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10)); + return (new PatternSummary(mappedGroups(table, estimating ? filter : DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10), records.size(), truncated)); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java index 2560174..14a8a1d 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java @@ -70,7 +70,7 @@ public class TableProfiler String name = field.getName(); AnalysisBackend.FieldCounts c = counts.get(name); result.getFields().add(new FieldSummary().withField(name).withLabel(field.getLabel()).withType(field.getType().name()) - .withCompleteness(c.completeness()).withDistinct(c.distinct()).withUniqueness(c.uniqueness()) + .withCompleteness(AnalysisFormatting.ratio(c.completeness())).withDistinct(c.distinct()).withUniqueness(AnalysisFormatting.ratio(c.uniqueness())) .withConstant(c.constant()).withEmpty(c.blanks() == rows) .withNulls(ProfileSupport.count("nulls", c.nulls(), table, drillFilter, DrillFilters.nulls(name, field.getType().isStringLike()))) .withBlanks(ProfileSupport.count("blanks", c.blanks(), table, drillFilter, c.blankFilter()))); diff --git a/src/main/resources/openapi/qqq-v1/data-analysis.yaml b/src/main/resources/openapi/qqq-v1/data-analysis.yaml index 7f0b3df..de5cdc6 100644 --- a/src/main/resources/openapi/qqq-v1/data-analysis.yaml +++ b/src/main/resources/openapi/qqq-v1/data-analysis.yaml @@ -511,7 +511,7 @@ components: nullable: true type: "string" completeness: - description: "Completeness." + description: "Non-empty share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" distinct: @@ -519,7 +519,7 @@ components: nullable: true type: "integer" uniqueness: - description: "Uniqueness." + description: "Distinct share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" constant: @@ -567,11 +567,13 @@ components: nullable: true type: "string" label: - description: "Label." + description: "Plain-English metric label, ready to display." nullable: true type: "string" value: - description: "Value." + description: "Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded\ + \ to six decimals; DECIMAL field statistics have ten significant digits.\ + \ DATE_TIME is an ISO instant." nullable: true oneOf: - type: "string" @@ -596,7 +598,8 @@ components: \ remains exact." type: "boolean" formatted: - description: "Formatted." + description: "Display-ready value, including grouping, field display formats,\ + \ percent signs and session-zone dates; null when absent or suppressed." nullable: true type: "string" drill: @@ -872,7 +875,7 @@ components: nullable: true type: "integer" share: - description: "Share." + description: "Share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" drill: diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java new file mode 100644 index 0000000..f454bee --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java @@ -0,0 +1,158 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting; +import com.kingsrook.qbits.dataanalysis.engine.MetricLabels; +import com.kingsrook.qbits.dataanalysis.engine.NiceBins; +import com.kingsrook.qbits.dataanalysis.engine.ProfileSupport; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/******************************************************************************* + ** Public presentation contracts, including precision and calendar edge cases. + *******************************************************************************/ +class AnalysisFormattingTest +{ + /******************************************************************************* + ** Raw ratios remain numeric while display strings are ready for a CLI. + *******************************************************************************/ + @Test + void numbersPercentagesAndLengths() + { + assertThat(ProfileSupport.metric("rows", 12498L, ValueType.COUNT).getFormatted()).isEqualTo("12,498"); + for(String key : List.of("uniqueness", "completeness", "share")) + { + var metric = ProfileSupport.metric(key, new BigDecimal("0.987654321"), ValueType.PERCENT); + assertThat(metric.getValue()).isEqualTo(new BigDecimal("0.987654")); + assertThat(metric.getFormatted()).isEqualTo("98.8%"); + assertThat(metric.getValueType()).isEqualTo(ValueType.PERCENT); + } + assertThat(ProfileSupport.metric("share", BigDecimal.ONE, ValueType.PERCENT).getFormatted()).isEqualTo("100%"); + assertThat(ProfileSupport.metric("share", BigDecimal.ZERO, ValueType.PERCENT).getFormatted()).isEqualTo("0%"); + assertThat(ProfileSupport.share(1, 3)).isEqualByComparingTo("0.333333"); + assertThat(ProfileSupport.share(0, 0)).isZero(); + assertThat(ProfileSupport.metric("meanLength", new BigDecimal("12.345"), ValueType.DECIMAL).getFormatted()).isEqualTo("12.3"); + assertThat(ProfileSupport.metric("minLength", BigDecimal.TEN, ValueType.DECIMAL).getFormatted()).isEqualTo("10"); + assertThat(ProfileSupport.metric("max", null, ValueType.DECIMAL).getFormatted()).isNull(); + assertThatThrownBy(() -> MetricLabels.label("unknown", ValueType.COUNT)).isInstanceOf(NullPointerException.class).hasMessageContaining("Missing metric label"); + } + + + + /******************************************************************************* + ** Host currency and custom display formats take precedence over numeric defaults. + *******************************************************************************/ + @Test + void fieldFormatsAndRawPrecision() + { + QFieldMetaData decimal = new QFieldMetaData("amount", QFieldType.DECIMAL); + assertThat(AnalysisFormatting.numeric(decimal, "min", new BigDecimal("81753.32"))).isEqualTo("81,753.32"); + decimal.withDisplayFormat(DisplayFormat.CURRENCY); + assertThat(AnalysisFormatting.numeric(decimal, "min", new BigDecimal("81753.32"))).isEqualTo("$81,753.32"); + var metric = AnalysisFormatting.numericMetric(decimal, "mean", new BigDecimal("125916.45167830173")); + assertThat(metric.getValue()).isEqualTo(new BigDecimal("125916.4517")); + assertThat(metric.getFormatted()).isEqualTo("$125,916.45"); + assertThat(AnalysisFormatting.numericMetric(decimal, "max", null).getValue()).isNull(); + QFieldMetaData integer = new QFieldMetaData("quantity", QFieldType.INTEGER); + assertThat(AnalysisFormatting.numeric(integer, "max", new BigDecimal("12498"))).isEqualTo("12,498"); + assertThat(AnalysisFormatting.numeric(integer, "mean", new BigDecimal("12.345"))).isEqualTo("12.35"); + assertThat(AnalysisFormatting.numeric(integer, "stddev", new BigDecimal("12.345"))).isEqualTo("12.35"); + assertThat(AnalysisFormatting.numeric(integer.withDisplayFormat(DisplayFormat.DECIMAL1_COMMAS), "mean", new BigDecimal("12.345"))).isEqualTo("12.3"); + QFieldMetaData large = new QFieldMetaData("large", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS); + assertThat(AnalysisFormatting.numeric(large, "sum", new BigDecimal("9000000000"))).isEqualTo("9,000,000,000"); + } + + + + /******************************************************************************* + ** Nice boundaries remain exact at small scales, negative values and constants. + *******************************************************************************/ + @Test + void niceEdgesAndLabels() + { + for(List range : List.of(List.of("25317.13", "65781.79"), List.of("-0.0093", "0.0357"), List.of("42", "42"), List.of("0", "0"), List.of("-9", "905"))) + { + BigDecimal low = new BigDecimal(range.getFirst()); + BigDecimal high = new BigDecimal(range.getLast()); + NiceBins bins = NiceBins.covering(low, high); + assertThat(bins.count()).isBetween(10, 25); + assertThat(bins.low()).isLessThanOrEqualTo(low); + assertThat(bins.high()).isGreaterThanOrEqualTo(high); + BigDecimal step = bins.high().subtract(bins.low()).divide(BigDecimal.valueOf(bins.count())).stripTrailingZeros(); + assertThat(step.scaleByPowerOfTen(step.scale() - step.precision() + 1).toPlainString()).isIn("1", "2", "2.5", "5"); + assertThat(bins.low().remainder(step)).isZero(); + assertThat(bins.high().remainder(step)).isZero(); + } + QFieldMetaData field = new QFieldMetaData("amount", QFieldType.INTEGER); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("60000"), new BigDecimal("65000"))).isEqualTo("60,000 – 65,000"); + assertThat(AnalysisFormatting.interval(field, null, new BigDecimal("25000"))).isEqualTo("Below 25,000"); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("65000"), null)).isEqualTo("65,000 and above"); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("1.25"), new BigDecimal("1.5"))).isEqualTo("1.25 – 1.5"); + field.withDisplayFormat(DisplayFormat.CURRENCY); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("60000"), new BigDecimal("65000"))).isEqualTo("$60,000 – $65,000"); + } + + + + /******************************************************************************* + ** Every supported unit has a human label without replacing its ISO coordinate. + *******************************************************************************/ + @Test + void dateUnitsAndDurations() + { + ZoneId zone = ZoneId.of("America/Chicago"); + LocalDate date = LocalDate.of(2024, 3, 4); + Map labels = Map.of( + AnalysisBackend.DateUnit.YEAR, "2024", AnalysisBackend.DateUnit.QUARTER, "2024 Q1", AnalysisBackend.DateUnit.MONTH, "Mar 2024", + AnalysisBackend.DateUnit.WEEK, "Week of 2024-03-04", AnalysisBackend.DateUnit.DAY, "2024-03-04"); + labels.forEach((unit, label) -> assertThat(AnalysisFormatting.dateBucket(date, unit, zone)).isEqualTo(label)); + assertThat(AnalysisFormatting.dateBucket(Instant.parse("2024-03-04T19:00:00Z"), AnalysisBackend.DateUnit.HOUR, zone)).isEqualTo("2024-03-04 13:00"); + for(int i = 1; i <= 7; i++) + { + assertThat(AnalysisFormatting.dateBucket(i, AnalysisBackend.DateUnit.DAY_OF_WEEK, zone)) + .isEqualTo(List.of("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday").get(i - 1)); + } + for(int i = 0; i < 24; i++) + { + assertThat(AnalysisFormatting.dateBucket(i, AnalysisBackend.DateUnit.HOUR_OF_DAY, zone)).isEqualTo(String.format(java.util.Locale.US, "%02d:00", i)); + } + assertThat(AnalysisFormatting.duration(date, date.plusYears(48).plusMonths(2))).isEqualTo("48 years 2 months"); + assertThat(AnalysisFormatting.duration(date, date.plusDays(12))).isEqualTo("12 days"); + assertThat(AnalysisFormatting.duration(date, date.plusYears(1).plusMonths(1).plusDays(1))).isEqualTo("1 year 1 month 1 day"); + assertThat(AnalysisFormatting.duration(date, date)).isEqualTo("0 days"); + assertThat(ProfileSupport.metric("min", date, ValueType.DATE).getLabel()).isEqualTo("Earliest"); + assertThat(ProfileSupport.metric("max", date, ValueType.DATE).getLabel()).isEqualTo("Latest"); + assertThat(ProfileSupport.metric("min", date, ValueType.DATE).getFormatted()).isEqualTo("2024-03-04"); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java index 324d20e..ed2a248 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java @@ -110,7 +110,7 @@ class CoreAnalysisBackendTest extends EngineFixture } /******************************************************************************* - ** Zero and capped percentages never divide by zero or restrict the row drill. + ** Empty samples keep the population drill; inspecting every row is exact. *******************************************************************************/ @Test void samplingBoundaries() throws Exception @@ -123,7 +123,8 @@ class CoreAnalysisBackendTest extends EngineFixture config.withDefaultSampleRows(100); service.getConfigResolver().invalidate(); var scope = service.backend("orders").scope("orders", null, false); - assertThat(scope.samplePercent()).isEqualByComparingTo("100"); + assertThat(scope.samplePercent()).isNull(); + assertThat(scope.estimate()).isFalse(); assertThat(scope.rowsScanned()).isEqualTo(12); } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java index 68c7fd9..f8439b7 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java @@ -33,7 +33,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; /******************************************************************************* @@ -62,8 +61,12 @@ class DataAnalysisHomeTest QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); String html = DataAnalysisHome.render(Instant.now()); assertThat(html).contains("Mine: 0", "Starred by me: 0", "No saved analyses yet", - "No active watches", "No completed or failed runs", "View all"); - assertThat(new DataAnalysisHome().render(new RenderWidgetInput())).isNotNull(); + "No watches yet.", "No analysis runs yet.", "View all", "Open any table and choose Analyze on a column header to explore its data.", "

"); + var widget = new DataAnalysisHome().render(new RenderWidgetInput()).getWidgetData(); + assertThat(widget).isInstanceOf(com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML.class); + assertThat(((com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML) widget).getType()).isEqualTo("html"); + assertThat(html).contains("·").doesNotContain("Â", "later", "phase", " c < 128)).isTrue(); QContext.init(instance, TestSupport.user("alice")); html = DataAnalysisHome.render(Instant.now()); assertThat(html).contains("you need data_analysis.saved.read", "you need data_analysis.watch.read").doesNotContain("Mine: 0", "Complete: 0"); @@ -99,11 +102,39 @@ class DataAnalysisHomeTest run("bob", "FAILED", now); QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); String html = DataAnalysisHome.render(now); - assertThat(html).contains("Mine: 2", "Shared (visible to you): 1", "Starred by me: 1", "Enabled: 2", "Firing: 1", "Complete: 2", "Failed: 1") - .doesNotContain("No saved analyses", "No active watches", "No completed or failed runs"); + assertThat(html).contains("Mine: 2", "Shared (visible to you): 1", "Starred by me: 1", "Enabled: 2", "Firing: 1", "Recent analysis runs", "Complete", "Failed", "Running", "Not finished") + .doesNotContain("No saved analyses", "No watches yet", "No analysis runs yet"); QContext.init(instance, TestSupport.user("bob").withPermissions("data_analysis.saved.read", "data_analysis.run.read")); html = DataAnalysisHome.render(now); - assertThat(html).contains("Mine: 1", "Shared (visible to you): 1", "Starred by me: 0", "Complete: 0", "Failed: 1", "you need data_analysis.watch.read"); + assertThat(html).contains("Mine: 1", "Shared (visible to you): 1", "Starred by me: 0", "Failed", "you need data_analysis.watch.read"); + } + + + + /******************************************************************************* + ** Latest five remain owner-scoped even with administrator bypass and escape HTML. + *******************************************************************************/ + @Test + void recentRunDetailsAndLimit() throws Exception + { + QInstance instance = TestSupport.instance(); + instance.getTable(DataAnalysisTables.SAVED).withLabel("Saved & review"); + Instant now = Instant.parse("2026-10-13T23:10:00Z"); + for(int i = 0; i < 7; i++) + { + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", DataAnalysisTables.SAVED) + .withValue("operation", "COLUMN_PROFILE").withValue("user_id", "admin").withValue("status", "COMPLETE") + .withValue("started_at", now.minusSeconds(i * 60L)).withValue("finished_at", now.minusSeconds(i * 60L))); + } + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "private_bob_table") + .withValue("operation", "TABLE_PROFILE").withValue("user_id", "bob").withValue("status", "FAILED").withValue("started_at", now)); + QContext.init(instance, TestSupport.admin().withValue(QSession.VALUE_KEY_USER_TIMEZONE, "America/Chicago")); + String html = DataAnalysisHome.render(now); + assertThat(html).contains("Saved <analysis> & review", "Column Profile", "2026-10-13 18:10", "2026-10-13 18:06", ""); + assertThat(html.split("
  • ", -1)).hasSize(6); + assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "/1\""); + assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "\">View all"); } @@ -113,7 +144,7 @@ class DataAnalysisHomeTest *******************************************************************************/ private static void run(String user, String status, Instant started) throws Exception { - TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "orders").withValue("operation", "profile") + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "orders").withValue("operation", "COLUMN_PROFILE") .withValue("user_id", user).withValue("status", status).withValue("started_at", started)); } } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java index c5343e5..3275715 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java @@ -64,7 +64,60 @@ import static org.assertj.core.api.Assertions.within; abstract class EngineContractTest extends EngineFixture { /******************************************************************************* - ** All numeric moments and percentiles agree to 1e-9. + ** Every emitted key across all profile kinds has a label, on every backend. + *******************************************************************************/ + @Test + void presentationAcrossProfileKinds() throws Exception + { + java.util.Set keys = new java.util.HashSet<>(); + for(String field : List.of("amount", "quantity", "note", "paid", "order_date", "created")) + { + ColumnProfile profile = service.columnProfile(column(field)); + java.util.List metrics = new java.util.ArrayList<>(profile.getCounts()); + if(profile.getNumeric() != null) + { + metrics.addAll(profile.getNumeric().getMetrics()); + } + if(profile.getText() != null) + { + metrics.addAll(profile.getText().getMetrics()); + } + if(profile.getDates() != null) + { + metrics.addAll(profile.getDates().getMetrics()); + assertThat(profile.getChart().getX().getLabel()).endsWith(" (day)"); + assertThat(profile.getChart().getSeries().getFirst().getPoints().getFirst().getX().toString()).isEqualTo("2026-01-01"); + } + if(profile.getBooleans() != null) + { + metrics.addAll(profile.getBooleans()); + } + for(Metric metric : metrics) + { + keys.add(metric.getKey()); + assertThat(metric.getLabel()).isEqualTo(com.kingsrook.qbits.dataanalysis.engine.MetricLabels.label(metric.getKey(), metric.getValueType())); + assertThat(metric.getFormatted()).isNotBlank(); + } + } + assertThat(keys).containsExactlyInAnyOrder("rows", "nonNull", "nulls", "blanks", "distinct", "uniqueness", "min", "max", "mean", "sum", "stddev", + "p1", "p5", "p25", "p50", "p75", "p95", "p99", "zeros", "negatives", "outliers", "minLength", "maxLength", "meanLength", "true", "false", "range"); + ColumnProfile amount = service.columnProfile(column("amount")); + assertThat(metric(amount.getNumeric().getMetrics(), "sum").getFormatted()).isEqualTo("$1,170.00"); + assertThat(amount.getNumeric().getHistogram()).allSatisfy(v -> assertThat(v.getLabel()).contains("$")); + Metric uniqueness = metric(amount.getCounts(), "uniqueness"); + assertThat(uniqueness.getValueType().name()).isEqualTo("PERCENT"); + assertThat(uniqueness.getValue().toString()).isEqualTo("0.727273"); + assertThat(uniqueness.getFormatted()).isEqualTo("72.7%"); + assertThat(metric(service.columnProfile(column("created")).getDates().getMetrics(), "min").getFormatted()).isEqualTo("2026-01-01 06:00"); + assertThat(service.columnProfile(column("note")).getWarnings()).isEmpty(); + var table = service.tableProfile(new TableProfileRequest().withTableName("orders").withFields(List.of("note"))); + assertThat(table.getRowCount().getLabel()).isEqualTo("Rows"); + assertThat(table.getFields().getFirst().getNulls().getLabel()).isEqualTo("Nulls"); + assertThat(table.getFields().getFirst().getBlanks().getLabel()).isEqualTo("Blanks"); + } + + /******************************************************************************* + ** Decimal moments retain ten significant digits; drills retain exact membership. *******************************************************************************/ @Test void numericProfile() throws Exception @@ -80,12 +133,12 @@ abstract class EngineContractTest extends EngineFixture Map.entry("zeros", "2"), Map.entry("negatives", "1"), Map.entry("outliers", "1")); for(var entry : expected.entrySet()) { - assertMetric(result.getNumeric().getMetrics(), entry.getKey(), entry.getValue()); + assertMetric(result.getNumeric().getMetrics(), entry.getKey(), new BigDecimal(entry.getValue()).round(new java.math.MathContext(10)).toPlainString()); } double[] values = {-10, 0, 0, 10, 10, 20, 20, 30, 40, 50, 1000}; double mean = 1170.0 / 11; double variance = java.util.Arrays.stream(values).map(v -> Math.pow(v - mean, 2)).sum() / 10; - assertMetric(result.getNumeric().getMetrics(), "stddev", Double.toString(Math.sqrt(variance))); + assertMetric(result.getNumeric().getMetrics(), "stddev", BigDecimal.valueOf(Math.sqrt(variance)).round(new java.math.MathContext(10)).toPlainString()); assertThat(result.getNumeric().getHistogram()).hasSize(22); assertThat(result.getNumeric().getHistogram().stream().mapToLong(v -> v.getCount()).sum()).isEqualTo(11); assertThat(result.getNumeric().getOutliers()).singleElement().satisfies(v -> assertThat(new BigDecimal(v.getValue().toString())).isEqualByComparingTo("1000")); @@ -156,7 +209,7 @@ abstract class EngineContractTest extends EngineFixture assertThat(table.getFields()).anySatisfy(f -> { assertThat(f.getField()).isEqualTo("note"); - assertThat(f.getCompleteness().doubleValue()).isCloseTo(8.0 / 12, within(1e-9)); + assertThat(f.getCompleteness().doubleValue()).isEqualTo(0.666667); }); assertThat(table.getCandidateKeys()).anyMatch(k -> k.getFields().equals(List.of("id"))); var pairs = service.tableProfile(new TableProfileRequest().withTableName("orders").withFields(List.of("status", "quantity"))); @@ -409,7 +462,7 @@ abstract class EngineContractTest extends EngineFixture assertThat(ColumnProfiler.autoBucket(first, first.plusYears(2).plusDays(1))).isEqualTo(AnalysisBackend.DateUnit.MONTH); assertThat(ColumnProfiler.autoBucket(first, first.plusYears(10))).isEqualTo(AnalysisBackend.DateUnit.MONTH); assertThat(ColumnProfiler.autoBucket(first, first.plusYears(10).plusDays(1))).isEqualTo(AnalysisBackend.DateUnit.YEAR); - for(AnalysisBackend.DateUnit unit : List.of(AnalysisBackend.DateUnit.WEEK, AnalysisBackend.DateUnit.MONTH, AnalysisBackend.DateUnit.YEAR)) + for(AnalysisBackend.DateUnit unit : List.of(AnalysisBackend.DateUnit.WEEK, AnalysisBackend.DateUnit.MONTH, AnalysisBackend.DateUnit.QUARTER, AnalysisBackend.DateUnit.YEAR)) { var buckets = service.backend("orders").dateBuckets("orders", null, "order_date", unit, java.time.ZoneId.of("America/Chicago")); assertThat(buckets.stream().mapToLong(AnalysisBackend.GroupCount::count).sum()).isEqualTo(12); diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java index 8642f4f..8c4105a 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; @@ -96,7 +97,7 @@ abstract class EngineFixture .withPermissionRules(DataAnalysisTables.permissionRules("orders")) .withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("customer", QFieldType.INTEGER).withPossibleValueSourceName("customer")) .withField(new QFieldMetaData("status", QFieldType.STRING).withPossibleValueSourceName("statusValues")) - .withField(new QFieldMetaData("amount", QFieldType.DECIMAL)).withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) + .withField(new QFieldMetaData("amount", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY)).withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) .withField(new QFieldMetaData("order_date", QFieldType.DATE)).withField(new QFieldMetaData("created", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("note", QFieldType.STRING)).withField(new QFieldMetaData("paid", QFieldType.BOOLEAN)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java new file mode 100644 index 0000000..b4a7ec4 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + +/******************************************************************************* + ** Real rows prove warning boundaries and date coordinates without Docker. + *******************************************************************************/ +class PresentationSamplingTest extends EngineFixture +{ + /******************************************************************************* + ** Null prefixes do not consume the budget; warn only after 10,000 values. + *******************************************************************************/ + @Test + void patternWarningOnlyWhenTruncated() throws Exception + { + List records = new ArrayList<>(); + for(int i = 100; i <= 10101; i++) + { + records.add(new QRecord().withValue("id", i).withValue("customer", 1).withValue("note", i == 100 ? null : "AA-11")); + } + assertThat(new InsertAction().execute(new InsertInput("orders").withRecords(records)).getRecords()).allSatisfy(r -> assertThat(r.getErrors()).isNullOrEmpty()); + var filter = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 100); + var request = new ColumnProfileRequest().withTableName("orders").withFieldName("note").withFilter(filter).withExact(true); + var truncated = service.columnProfile(request); + assertThat(truncated.getText().getSampleRows()).isEqualTo(10000); + assertThat(truncated.getWarnings()).containsExactly("Patterns are based on the first 10,000 values."); + request.withFilter(DrillFilters.and(filter, DrillFilters.condition("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, 10100))); + var all = service.columnProfile(request); + assertThat(all.getText().getSampleRows()).isEqualTo(10000); + assertThat(all.getWarnings()).isEmpty(); + } + + + + /******************************************************************************* + ** Empty samples are not empty populations; full scans are not estimates. + *******************************************************************************/ + @Test + void warningsDescribeObservedScope() throws Exception + { + config.withDefaultSampleThresholdRows(1).withDefaultSampleRows(100); + var request = new ColumnProfileRequest().withTableName("orders").withFieldName("note"); + var complete = service.columnProfile(request); + assertThat(complete.getEstimate()).isFalse(); + assertThat(complete.getWarnings()).isEmpty(); + config.withDefaultSampleRows(0); + service.getConfigResolver().invalidate(); + var emptySample = service.columnProfile(request); + assertThat(emptySample.getWarnings()).contains("No sample rows were observed; run exact for exact numbers.") + .doesNotContain("No readable records match this analysis scope.", "All values in this analysis scope are null."); + var empty = service.columnProfile(request.withExact(true).withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, -1))); + assertThat(empty.getWarnings()).containsExactly("No readable records match this analysis scope."); + var nulls = service.columnProfile(request.withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, 1))); + assertThat(nulls.getWarnings()).containsExactly("All values in this analysis scope are null."); + } + + + + /******************************************************************************* + ** Hour buckets cross DST in real elapsed hours and preserve ISO instants. + *******************************************************************************/ + @Test + void hourlyBucketCoordinatesAndLabels() throws Exception + { + var backend = service.backend("orders"); + ZoneId zone = ZoneId.of("America/Chicago"); + var buckets = backend.dateBuckets("orders", null, "created", AnalysisBackend.DateUnit.HOUR, zone); + assertThat(buckets.getFirst().value()).isEqualTo(Instant.parse("2026-01-01T12:00:00Z")); + assertThat(buckets.getFirst().label()).isEqualTo("2026-01-01 06:00"); + assertThat(buckets.stream().mapToLong(AnalysisBackend.GroupCount::count).sum()).isEqualTo(12); + for(var bucket : buckets) + { + assertThat(backend.count("orders", bucket.condition())).isEqualTo(bucket.count()); + } + TestSupport.insert("orders", new QRecord().withValue("id", 20).withValue("customer", 1).withValue("created", Instant.parse("2026-11-01T06:30:00Z"))); + TestSupport.insert("orders", new QRecord().withValue("id", 21).withValue("customer", 1).withValue("created", Instant.parse("2026-11-01T07:30:00Z"))); + var filter = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 20); + var dst = backend.dateBuckets("orders", filter, "created", AnalysisBackend.DateUnit.HOUR, zone); + assertThat(dst).hasSize(2).allSatisfy(b -> assertThat(b.label()).isEqualTo("2026-11-01 01:00")); + assertThat(dst).extracting(AnalysisBackend.GroupCount::value).doesNotHaveDuplicates(); + assertThat(dst).allSatisfy(b -> assertThat(b.count()).isEqualTo(1)); + } +}