diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java index 794dbd5..151c72e 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/api/DataAnalysisSchemas.java @@ -603,13 +603,13 @@ public class DataAnalysisSchemas @OpenAPIDescription("Type.") public String type; - @OpenAPIDescription("Completeness.") + @OpenAPIDescription("Non-empty share as a 0..1 ratio rounded to six decimals.") public BigDecimal completeness; @OpenAPIDescription("Distinct.") public Long distinct; - @OpenAPIDescription("Uniqueness.") + @OpenAPIDescription("Distinct share as a 0..1 ratio rounded to six decimals.") public BigDecimal uniqueness; @OpenAPIDescription("Constant.") @@ -634,9 +634,9 @@ public class DataAnalysisSchemas properties.put("field", new Schema().withType(Type.STRING).withDescription("Field.").withNullable(true)); properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); properties.put("type", new Schema().withType(Type.STRING).withDescription("Type.").withNullable(true)); - properties.put("completeness", new Schema().withType(Type.NUMBER).withDescription("Completeness.").withNullable(true)); + properties.put("completeness", new Schema().withType(Type.NUMBER).withDescription("Non-empty share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("distinct", new Schema().withType(Type.INTEGER).withDescription("Distinct.").withNullable(true)); - properties.put("uniqueness", new Schema().withType(Type.NUMBER).withDescription("Uniqueness.").withNullable(true)); + properties.put("uniqueness", new Schema().withType(Type.NUMBER).withDescription("Distinct share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("constant", new Schema().withType(Type.BOOLEAN).withDescription("Constant.").withNullable(true)); properties.put("empty", new Schema().withType(Type.BOOLEAN).withDescription("Empty.").withNullable(true)); properties.put("nulls", ref("DataAnalysisMetricV1").withDescription("Nulls.").withNullable(true)); @@ -654,16 +654,16 @@ public class DataAnalysisSchemas public Boolean estimate; public String key; - @OpenAPIDescription("Label.") + @OpenAPIDescription("Plain-English metric label, ready to display.") public String label; - @OpenAPIDescription("Value.") + @OpenAPIDescription("Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded to six decimals; DECIMAL field statistics have ten significant digits. DATE_TIME is an ISO instant.") public Serializable value; @OpenAPIDescription("Value type.") public ValueType valueType; - @OpenAPIDescription("Formatted.") + @OpenAPIDescription("Display-ready value, including grouping, field display formats, percent signs and session-zone dates; null when absent or suppressed.") public String formatted; @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") @@ -683,11 +683,11 @@ public class DataAnalysisSchemas { Map properties = new LinkedHashMap<>(); properties.put("key", new Schema().withType(Type.STRING).withDescription("Key.").withNullable(true)); - properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); - properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Value.").withNullable(true)); + properties.put("label", new Schema().withType(Type.STRING).withDescription("Plain-English metric label, ready to display.").withNullable(true)); + properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded to six decimals; DECIMAL field statistics have ten significant digits. DATE_TIME is an ISO instant.").withNullable(true)); properties.put("valueType", new Schema().withType(Type.STRING).withEnumValues(java.util.Arrays.stream(ValueType.values()).map(Enum::name).toList()).withDescription("Value type.").withNullable(true)); properties.put("estimate", new Schema().withType(Type.BOOLEAN).withDescription("Whether this metric is estimated; the population rows metric remains exact.")); - properties.put("formatted", new Schema().withType(Type.STRING).withDescription("Formatted.").withNullable(true)); + properties.put("formatted", new Schema().withType(Type.STRING).withDescription("Display-ready value, including grouping, field display formats, percent signs and session-zone dates; null when absent or suppressed.").withNullable(true)); properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); properties.put("suppressed", new Schema().withType(Type.BOOLEAN).withDescription("Suppressed.").withNullable(true)); properties.put("suppressedReason", new Schema().withType(Type.STRING).withDescription("Suppressed reason.").withNullable(true)); @@ -961,7 +961,7 @@ public class DataAnalysisSchemas @OpenAPIDescription("Count.") public Long count; - @OpenAPIDescription("Share.") + @OpenAPIDescription("Share as a 0..1 ratio rounded to six decimals.") public BigDecimal share; @OpenAPIDescription("Open the matching records using this table and QueryFilter; null when suppressed.") @@ -977,7 +977,7 @@ public class DataAnalysisSchemas properties.put("value", new Schema().withOneOf(List.of(new Schema().withType(Type.STRING), new Schema().withType(Type.NUMBER), new Schema().withType(Type.BOOLEAN), new Schema().withType(Type.OBJECT).withNullable(true))).withDescription("Value.").withNullable(true)); properties.put("label", new Schema().withType(Type.STRING).withDescription("Label.").withNullable(true)); properties.put("count", new Schema().withType(Type.INTEGER).withDescription("Count.").withNullable(true)); - properties.put("share", new Schema().withType(Type.NUMBER).withDescription("Share.").withNullable(true)); + properties.put("share", new Schema().withType(Type.NUMBER).withDescription("Share as a 0..1 ratio rounded to six decimals.").withNullable(true)); properties.put("drill", ref("DataAnalysisDrillV1").withDescription("Open the matching records using this table and QueryFilter; null when suppressed.").withNullable(true)); return (new Schema().withType(Type.OBJECT).withNullable(true).withProperties(properties)); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java index b8d0bb8..fa345a9 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/app/DataAnalysisHome.java @@ -22,26 +22,33 @@ package com.kingsrook.qbits.dataanalysis.app; import java.time.Instant; -import java.time.temporal.ChronoUnit; +import java.util.Arrays; import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; import com.kingsrook.qbits.dataanalysis.DataAnalysisQBitConfig; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting; import com.kingsrook.qbits.dataanalysis.model.DataAnalysisTables; import com.kingsrook.qqq.backend.core.actions.dashboard.widgets.AbstractWidgetRenderer; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionEnforcement; import com.kingsrook.qqq.backend.core.actions.permissions.PermissionsHelper; import com.kingsrook.qqq.backend.core.actions.permissions.TablePermissionSubType; import com.kingsrook.qqq.backend.core.actions.tables.CountAction; +import com.kingsrook.qqq.backend.core.actions.tables.QueryAction; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.model.actions.tables.count.CountInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterCriteria; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryJoin; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetInput; import com.kingsrook.qqq.backend.core.model.actions.widgets.RenderWidgetOutput; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML; import com.kingsrook.qqq.backend.core.model.dashboard.widgets.WidgetType; +import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import com.kingsrook.qqq.backend.core.model.metadata.dashboard.QWidgetMetaData; import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; @@ -93,7 +100,8 @@ public class DataAnalysisHome extends AbstractWidgetRenderer /******************************************************************************* - ** Standard QQQ widget renderer entry point. + ** RawHTML is carried as a JSON string by the HTML widget, not a text/html response. + ** Entities keep static separators independent of any host charset handling. *******************************************************************************/ @Override public RenderWidgetOutput render(RenderWidgetInput input) throws QException @@ -110,7 +118,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { StringBuilder html = new StringBuilder("
"); String userId = QContext.getQSession() == null || QContext.getQSession().getUser() == null ? null : QContext.getQSession().getUser().getIdReference(); - html.append("

Saved analyses

"); + html.append("

Saved analyses

"); if(userId == null) { html.append("

Sign in to see your saved analyses and stars.

"); @@ -119,7 +127,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { int mine = count(DataAnalysisTables.SAVED, equal("owner_user_id", userId)); int shared = count(DataAnalysisTables.SAVED, equal("shared", true)); - html.append("

Mine: ").append(mine).append(" · Shared (visible to you): ").append(shared).append("

"); + html.append("

Mine: ").append(AnalysisFormatting.number(mine, 0)).append(" · Shared (visible to you): ").append(AnalysisFormatting.number(shared, 0)).append("

"); if(canRead(DataAnalysisTables.SAVED_STAR)) { CountInput stars = new CountInput(DataAnalysisTables.SAVED).withIncludeDistinctCount(true) @@ -127,28 +135,28 @@ public class DataAnalysisHome extends AbstractWidgetRenderer .withFilter(equal(DataAnalysisTables.SAVED_STAR + ".user_id", userId)); stars.setPermissionEnforcement(PermissionEnforcement.ENFORCE); int starred = new CountAction().execute(stars).getDistinctCount(); - html.append("

Starred by me: ").append(starred).append("

"); + html.append("

Starred by me: ").append(AnalysisFormatting.number(starred, 0)).append("

"); } if(mine == 0 && shared == 0) { - html.append("

No saved analyses yet. Saving analyses from the Analyze action is planned for a later release.

"); + html.append("

No saved analyses yet. Open any table and choose Analyze on a column header to explore its data.

"); } - html.append("

The Analyze action will be available in a later UI release.

"); link(html, DataAnalysisTables.SAVED); } else { html.append("

Saved analyses are unavailable: you need data_analysis.saved.read.

"); } - html.append("

Watches

"); + html.append("

Watches

"); if(canRead(DataAnalysisTables.WATCH)) { + int total = count(DataAnalysisTables.WATCH, new QQueryFilter()); int enabled = count(DataAnalysisTables.WATCH, equal("enabled", true)); int firing = count(DataAnalysisTables.WATCH, equal("state", "FIRING")); - html.append("

Enabled: ").append(enabled).append(" · Firing: ").append(firing).append("

"); - if(enabled == 0 && firing == 0) + html.append("

Total: ").append(AnalysisFormatting.number(total, 0)).append(" · Enabled: ").append(AnalysisFormatting.number(enabled, 0)).append(" · Firing: ").append(AnalysisFormatting.number(firing, 0)).append("

"); + if(total == 0) { - html.append("

No active watches: watch evaluation is planned for phase e.

"); + html.append("

No watches yet.

"); } link(html, DataAnalysisTables.WATCH); } @@ -156,15 +164,37 @@ public class DataAnalysisHome extends AbstractWidgetRenderer { html.append("

Watches are unavailable: you need data_analysis.watch.read.

"); } - html.append("

Runs in the last 7 days

"); - if(canRead(DataAnalysisTables.RUN)) + html.append("

Recent analysis runs

"); + if(userId == null) { - int complete = count(DataAnalysisTables.RUN, recentRuns("COMPLETE", now)); - int failed = count(DataAnalysisTables.RUN, recentRuns("FAILED", now)); - html.append("

Complete: ").append(complete).append(" · Failed: ").append(failed).append("

"); - if(complete == 0 && failed == 0) + html.append("

Sign in to see your recent analysis runs.

"); + } + else if(canRead(DataAnalysisTables.RUN)) + { + QueryInput query = new QueryInput(DataAnalysisTables.RUN).withFilter(equal("user_id", userId) + .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now)) + .withOrderBy(new QFilterOrderBy("started_at", false)).withOrderBy(new QFilterOrderBy("id", false)).withLimit(5)); + query.setPermissionEnforcement(PermissionEnforcement.ENFORCE); + List runs = new QueryAction().execute(query).getRecords(); + if(runs.isEmpty()) { - html.append("

No completed or failed runs in this period.

"); + html.append("

No analysis runs yet.

"); + } + else + { + html.append("
    "); + for(QRecord run : runs) + { + QTableMetaData table = QContext.getQInstance().getTable(run.getValueString("table_name")); + String tableLabel = table == null ? run.getValueString("table_name") : table.getLabel(); + String path = QContext.getQInstance().getTablePath(DataAnalysisTables.RUN) + "/" + run.getValueInteger("id"); + html.append("
  • ").append(escape(tableLabel)).append(" · ") + .append(escape(words(run.getValueString("operation")))).append("
    Status
    ") + .append(escape(words(run.getValueString("status")))).append("
    Finished
    ") + .append(run.getValue("finished_at") == null ? "Not finished" : escape(AnalysisFormatting.instant(run.getValueInstant("finished_at")))) + .append("
  • "); + } + html.append("
"); } link(html, DataAnalysisTables.RUN); } @@ -198,13 +228,22 @@ public class DataAnalysisHome extends AbstractWidgetRenderer /******************************************************************************* - ** Runs started within the inclusive seven-day interval, excluding future rows. + ** Stored operation and status keys are displayed as ordinary words. *******************************************************************************/ - private static QQueryFilter recentRuns(String status, Instant now) + private static String words(String value) { - return (equal("status", status) - .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.GREATER_THAN_OR_EQUALS, now.minus(7, ChronoUnit.DAYS))) - .withCriteria(new QFilterCriteria("started_at", QCriteriaOperator.LESS_THAN_OR_EQUALS, now))); + return (value == null ? "Unknown" : Arrays.stream(value.toLowerCase(Locale.ROOT).split("_")) + .filter(part -> !part.isEmpty()).map(part -> Character.toUpperCase(part.charAt(0)) + part.substring(1)).collect(Collectors.joining(" "))); + } + + + + /******************************************************************************* + ** Metadata and persisted values must never become executable widget markup. + *******************************************************************************/ + private static String escape(String value) + { + return (value == null ? "Unknown" : value.replace("&", "&").replace("\"", """).replace("<", "<").replace(">", ">").replace("'", "'")); } @@ -227,7 +266,7 @@ public class DataAnalysisHome extends AbstractWidgetRenderer private static void link(StringBuilder html, String table) throws QException { String path = QContext.getQInstance().getTablePath(table); - html.append("", ">")) + html.append("View all"); } } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java index 14ebe27..77f122c 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisBackend.java @@ -38,7 +38,7 @@ public interface AnalysisBackend *******************************************************************************/ enum DateUnit { - DAY, WEEK, MONTH, YEAR, DAY_OF_WEEK, HOUR_OF_DAY + DAY, WEEK, MONTH, QUARTER, YEAR, HOUR, DAY_OF_WEEK, HOUR_OF_DAY } @@ -120,6 +120,15 @@ public interface AnalysisBackend + /******************************************************************************* + ** Actual inspected non-null values, independent of estimated population counts. + *******************************************************************************/ + record PatternSummary(List groups, long values, boolean truncated) + { + } + + + /******************************************************************************* ** Filtered count; a native estimate scales conditional counts to its population. *******************************************************************************/ @@ -168,7 +177,7 @@ public interface AnalysisBackend /******************************************************************************* ** Top shapes over a bounded primary-key prefix. *******************************************************************************/ - List patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException; + PatternSummary patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException; /******************************************************************************* ** Top groups with count greater than one. diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java new file mode 100644 index 0000000..badae57 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/AnalysisFormatting.java @@ -0,0 +1,225 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.io.Serializable; +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.time.DayOfWeek; +import java.time.Instant; +import java.time.LocalDate; +import java.time.Period; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.time.format.TextStyle; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import com.kingsrook.qbits.dataanalysis.result.Metric; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.actions.values.QValueFormatter; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import com.kingsrook.qqq.backend.core.utils.ValueUtils; + +/******************************************************************************* + ** Display-ready values shared by metrics, chart labels and the home widget. + *******************************************************************************/ +public final class AnalysisFormatting +{ + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private AnalysisFormatting() + { + } + + + + /******************************************************************************* + ** Ratios remain numeric and bounded, with stable JSON precision. + *******************************************************************************/ + public static BigDecimal ratio(BigDecimal value) + { + return (value.max(BigDecimal.ZERO).min(BigDecimal.ONE).setScale(6, RoundingMode.HALF_UP).stripTrailingZeros()); + } + + + + /******************************************************************************* + ** Plain numeric defaults are independent of the server locale. + *******************************************************************************/ + public static String number(Number value, int decimals) + { + return (String.format(Locale.US, "%,." + decimals + "f", new BigDecimal(value.toString()))); + } + + + + /******************************************************************************* + ** Types and length keys determine display without changing numeric raw values. + *******************************************************************************/ + public static String scalar(String key, Serializable value, ValueType type) + { + if(value == null) + { + return (null); + } + if(type == ValueType.PERCENT) + { + BigDecimal percent = new BigDecimal(value.toString()).multiply(BigDecimal.valueOf(100)).setScale(1, RoundingMode.HALF_UP); + return (percent.signum() == 0 || percent.compareTo(BigDecimal.valueOf(100)) == 0 ? percent.toBigInteger() + "%" : percent.toPlainString() + "%"); + } + if(type == ValueType.DATE_TIME) + { + return (instant((Instant) value)); + } + if(value instanceof Number n) + { + return (number(n, "meanLength".equals(key) ? 1 : type == ValueType.COUNT || key.endsWith("Length") ? 0 : 2)); + } + return (value.toString()); + } + + + + /******************************************************************************* + ** Display formats belong to the host field; fallback defaults preserve fractions. + *******************************************************************************/ + public static String numeric(QFieldMetaData field, String key, BigDecimal value) + { + if(value == null) + { + return (null); + } + String format = field.getDisplayFormat(); + if(format != null && !format.isBlank() && !DisplayFormat.DEFAULT.equals(format)) + { + Serializable displayValue = format.endsWith("d") ? value.toBigInteger() : value; + return (QValueFormatter.formatValue(format, displayValue)); + } + return (number(value, field.getType() == QFieldType.DECIMAL || "mean".equals(key) || "stddev".equals(key) ? 2 : 0)); + } + + + + /******************************************************************************* + ** Decimal statistics retain ten significant digits; formatting uses the original. + *******************************************************************************/ + public static Metric numericMetric(QFieldMetaData field, String key, BigDecimal value) + { + BigDecimal raw = value == null || field.getType() != QFieldType.DECIMAL ? value : value.round(new MathContext(10)); + return (ProfileSupport.metric(key, raw, ValueType.DECIMAL).withFormatted(numeric(field, key, value))); + } + + + + /******************************************************************************* + ** Use the same field formatter on both ends of every histogram interval. + *******************************************************************************/ + public static String interval(QFieldMetaData field, BigDecimal low, BigDecimal high) + { + return (low == null ? "Below " + edge(field, high) + : high == null ? edge(field, low) + " and above" + : edge(field, low) + " – " + edge(field, high)); + } + + + + /******************************************************************************* + ** Whole edges omit redundant cents; unformatted fractional edges stay distinct. + *******************************************************************************/ + private static String edge(QFieldMetaData field, BigDecimal value) + { + String format = field.getDisplayFormat(); + int scale = Math.max(0, value.stripTrailingZeros().scale()); + if(format == null || format.isBlank() || DisplayFormat.DEFAULT.equals(format)) + { + return (number(value, scale)); + } + String formatted = numeric(field, "min", value); + return (scale == 0 && DisplayFormat.CURRENCY.equals(format) && formatted.endsWith(".00") + ? formatted.substring(0, formatted.length() - 3) : formatted); + } + + + + /******************************************************************************* + ** Instants remain UTC in raw values and display in the current session zone. + *******************************************************************************/ + public static String instant(Instant value) + { + return (DateTimeFormatter.ofPattern("uuuu-MM-dd HH:mm", Locale.US).format(value.atZone(ValueUtils.getSessionOrInstanceZoneId()))); + } + + + + /******************************************************************************* + ** Calendar duration avoids misleading fixed-length month and year arithmetic. + *******************************************************************************/ + public static String duration(LocalDate first, LocalDate last) + { + Period period = Period.between(first, last); + List parts = new ArrayList<>(); + if(period.getYears() != 0) + { + parts.add(period.getYears() + (period.getYears() == 1 ? " year" : " years")); + } + if(period.getMonths() != 0) + { + parts.add(period.getMonths() + (period.getMonths() == 1 ? " month" : " months")); + } + if(period.getDays() != 0 || parts.isEmpty()) + { + parts.add(period.getDays() + (period.getDays() == 1 ? " day" : " days")); + } + return (String.join(" ", parts)); + } + + + + /******************************************************************************* + ** Calendar labels describe one bucket; the raw value remains its ISO start. + *******************************************************************************/ + public static String dateBucket(Serializable start, AnalysisBackend.DateUnit unit, ZoneId zone) + { + if(unit == AnalysisBackend.DateUnit.DAY_OF_WEEK) + { + return (DayOfWeek.of(((Number) start).intValue()).getDisplayName(TextStyle.FULL, Locale.US)); + } + if(unit == AnalysisBackend.DateUnit.HOUR_OF_DAY) + { + return (String.format(Locale.US, "%02d:00", ((Number) start).intValue())); + } + LocalDate date = ProfileSupport.day(start, zone); + return (switch(unit) + { + case YEAR -> Integer.toString(date.getYear()); + case QUARTER -> date.getYear() + " Q" + ((date.getMonthValue() - 1) / 3 + 1); + case MONTH -> DateTimeFormatter.ofPattern("MMM uuuu", Locale.US).format(date); + case WEEK -> "Week of " + date; + case HOUR -> DateTimeFormatter.ofPattern("uuuu-MM-dd HH:00", Locale.US).format(((Instant) start).atZone(zone)); + default -> date.toString(); + }); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java index 55edeb3..27945b2 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ColumnProfiler.java @@ -86,7 +86,7 @@ public class ColumnProfiler QFieldType type = field.getType(); if(type == QFieldType.INTEGER || type == QFieldType.LONG || type == QFieldType.DECIMAL) { - numeric(result, filter, drillFilter, rows); + numeric(result, filter, drillFilter, rows, field); } else if(type == QFieldType.DATE || type == QFieldType.DATE_TIME) { @@ -95,13 +95,18 @@ public class ColumnProfiler else if(type.isStringLike()) { AnalysisBackend.LengthSummary summary = backend.lengthSummary(table, filter, name); + AnalysisBackend.PatternSummary patterns = backend.patterns(table, filter, name, 10_000); + long patternValues = patterns.values(); TextProfile text = new TextProfile().withHistogram(ProfileSupport.values(table, drillFilter, null, summary.histogram(), rows)) - .withPatterns(ProfileSupport.values(table, drillFilter, null, backend.patterns(table, filter, name, 10_000), Math.min(rows, 10_000))) - .withSampleRows(Math.min(scope.rowsScanned(), 10_000)); + .withPatterns(ProfileSupport.values(table, drillFilter, null, patterns.groups(), patternValues)) + .withSampleRows(patternValues); summary.metrics().forEach((key, value) -> text.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); result.withText(text); - result.getWarnings().add(scope.samplePercent() == null ? "Patterns use the first " + Math.min(rows, 10_000) + " rows by primary key." - : "Patterns use up to " + Math.min(scope.rowsScanned(), 10_000) + " sampled rows."); + if(patterns.truncated()) + { + result.getWarnings().add(scope.samplePercent() == null ? "Patterns are based on the first " + AnalysisFormatting.number(patternValues, 0) + " values." + : "Patterns are based on " + AnalysisFormatting.number(patternValues, 0) + " sampled values."); + } } else if(type == QFieldType.BOOLEAN) { @@ -142,14 +147,15 @@ public class ColumnProfiler else { result.withChart(ProfileSupport.chart(ChartKind.TIME_SERIES, ChartKind.BAR, name, result.getDates().getBuckets())); + result.getChart().getX().withLabel(field.getLabel() + (result.getDates().getUnit() == null ? "" : " (" + result.getDates().getUnit().toLowerCase(java.util.Locale.ROOT) + ")")); } - if(rows == 0) + if(scope.total() == 0) { result.getWarnings().add("No readable records match this analysis scope."); } - else if(counts.nonNull() == 0 && (!scope.estimate() || scope.rowsScanned() > 0)) + else if(counts.nonNull() == 0 && rows > 0 && (!scope.estimate() || scope.rowsScanned() > 0)) { - result.getWarnings().add("All values in this analysis scope are null."); + result.getWarnings().add(scope.estimate() ? "All observed values are null." : "All values in this analysis scope are null."); } if(scope.estimate()) { @@ -170,16 +176,16 @@ public class ColumnProfiler /******************************************************************************* - ** Numeric moments, twenty interior bins, exterior bins and Tukey outliers. + ** Numeric moments, nice interior bins, exterior bins and Tukey outliers. *******************************************************************************/ - private void numeric(ColumnProfile result, QQueryFilter filter, QQueryFilter drillFilter, long rows) throws QException + private void numeric(ColumnProfile result, QQueryFilter filter, QQueryFilter drillFilter, long rows, QFieldMetaData metadata) throws QException { String table = result.getTableName(); String field = result.getFieldName(); AnalysisBackend.NumericSummary summary = backend.numericSummary(table, filter, field, List.of("0.01", "0.05", "0.25", "0.50", "0.75", "0.95", "0.99").stream().map(BigDecimal::new).toList()); NumericProfile numeric = new NumericProfile(); - summary.metrics().forEach((key, value) -> numeric.getMetrics().add(ProfileSupport.metric(key, value, ValueType.DECIMAL))); + summary.metrics().forEach((key, value) -> numeric.getMetrics().add(AnalysisFormatting.numericMetric(metadata, key, value))); for(AnalysisBackend.GroupCount count : summary.counts()) { numeric.getMetrics().add(ProfileSupport.count(count.label(), count.count(), table, drillFilter, count.condition())); @@ -211,7 +217,7 @@ public class ColumnProfiler LocalDate last = ProfileSupport.day(extrema.get("max"), zone); AnalysisBackend.DateUnit unit = autoBucket(first, last); dates.withUnit(unit.name()); - dates.getMetrics().add(ProfileSupport.metric("range", ChronoUnit.DAYS.between(first, last), ValueType.DURATION)); + dates.getMetrics().add(ProfileSupport.metric("range", ChronoUnit.DAYS.between(first, last), ValueType.DURATION).withFormatted(AnalysisFormatting.duration(first, last))); List buckets = ProfileSupport.values(table, drillFilter, null, backend.dateBuckets(table, filter, field, unit, zone), rows); dates.withBuckets(buckets).withGaps(buckets.stream().filter(b -> b.getCount() == 0).toList()); if(type == QFieldType.DATE_TIME) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java index e5f9201..3520581 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/CoreAnalysisBackend.java @@ -85,7 +85,7 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend { samplePercent = null; long total = super.count(table, filter); - if(exact || total <= config.getSampleThresholdRows() || !capabilities(table).contains(AggregateCapability.SAMPLING)) + if(exact || total <= config.getSampleThresholdRows() || total <= config.getSampleRows() || !capabilities(table).contains(AggregateCapability.SAMPLING)) { return (super.scope(table, filter, exact)); } @@ -375,21 +375,25 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend { return (super.numericHistogram(table, filter, field, low, high, bins)); } + NiceBins nice = NiceBins.covering(low, high); + low = nice.low(); + high = nice.high(); + bins = nice.count(); GroupBy group = new GroupBy(field(table, field)).withBucket(new NumericBucket(low, high, bins)); Map counts = bucketCounts(table, filter, group); List values = !estimating && isMemory(table) ? rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList() : List.of(); List result = new ArrayList<>(); - result.add(new GroupCount("Below " + low, "Below " + low, aggregateCount(counts.getOrDefault(0, 0L)), numericRange(table, field, values, null, low, false), aggregateShare(counts.getOrDefault(0, 0L)))); + result.add(new GroupCount(AnalysisFormatting.interval(field(table, field), null, low), AnalysisFormatting.interval(field(table, field), null, low), aggregateCount(counts.getOrDefault(0, 0L)), numericRange(table, field, values, null, low, false), aggregateShare(counts.getOrDefault(0, 0L)))); BigDecimal width = high.subtract(low).divide(BigDecimal.valueOf(bins), MathContext.DECIMAL128); for(int i = 0; i < bins; i++) { BigDecimal start = low.add(width.multiply(BigDecimal.valueOf(i))); BigDecimal end = i == bins - 1 ? high : low.add(width.multiply(BigDecimal.valueOf(i + 1))); - String label = "[" + start + ", " + end + (i == bins - 1 ? "]" : ")"); + String label = AnalysisFormatting.interval(field(table, field), start, end); result.add(new GroupCount(label, label, aggregateCount(counts.getOrDefault(i + 1, 0L)), numericRange(table, field, values, start, end, i == bins - 1), aggregateShare(counts.getOrDefault(i + 1, 0L)))); } - result.add(new GroupCount("Above " + high, "Above " + high, aggregateCount(counts.getOrDefault(bins + 1, 0L)), + result.add(new GroupCount(AnalysisFormatting.interval(field(table, field), high, null), AnalysisFormatting.interval(field(table, field), high, null), aggregateCount(counts.getOrDefault(bins + 1, 0L)), DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)), aggregateShare(counts.getOrDefault(bins + 1, 0L)))); return (result); } @@ -400,6 +404,10 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend @Override public List dateBuckets(String table, QQueryFilter filter, String field, DateUnit unit, ZoneId zone) throws QException { + if(unit == DateUnit.HOUR || unit == DateUnit.QUARTER) + { + return (super.dateBuckets(table, filter, field, unit, zone)); + } GroupBy group = new GroupBy(field(table, field)).withBucket(new DateBucket(DateBucketUnit.valueOf(unit.name()))); AggregateInput probe = new AggregateInput(table).withGroupBy(group); if(!new QBackendModuleDispatcher().getQBackendModule(probe.getBackend()).getAggregateInterface().getSupportedCapabilities(probe).contains(AggregateCapability.DATE_BUCKET)) @@ -423,8 +431,8 @@ public class CoreAnalysisBackend extends StandInAnalysisBackend while(!start.isAfter(dates.getLast())) { LocalDate end = next(start, unit); - String label = start + " / " + end; - result.add(new GroupCount(label, label, aggregateCount(counts.getOrDefault(start, 0L)), + String label = AnalysisFormatting.dateBucket(start, unit, zone); + result.add(new GroupCount(start, label, aggregateCount(counts.getOrDefault(start, 0L)), DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone), aggregateShare(counts.getOrDefault(start, 0L)))); start = end; } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java new file mode 100644 index 0000000..2b7ce09 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MetricLabels.java @@ -0,0 +1,65 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.util.Map; +import java.util.Objects; +import com.kingsrook.qbits.dataanalysis.result.ValueType; + +/******************************************************************************* + ** One vocabulary for every emitted metric, including date-specific extrema. + *******************************************************************************/ +public final class MetricLabels +{ + private static final Map LABELS = Map.ofEntries( + Map.entry("rows", "Rows"), Map.entry("nonNull", "Non-empty"), Map.entry("nulls", "Nulls"), + Map.entry("blanks", "Blanks"), Map.entry("distinct", "Distinct values"), Map.entry("uniqueness", "Uniqueness"), + Map.entry("completeness", "Completeness"), Map.entry("share", "Share"), + Map.entry("min", "Minimum"), Map.entry("max", "Maximum"), Map.entry("mean", "Mean"), + Map.entry("median", "Median"), Map.entry("stddev", "Standard deviation"), Map.entry("sum", "Sum"), + Map.entry("zeros", "Zeros"), Map.entry("negatives", "Negatives"), Map.entry("outliers", "Outliers"), + Map.entry("p1", "1st percentile"), Map.entry("p5", "5th percentile"), Map.entry("p25", "25th percentile (Q1)"), + Map.entry("p50", "Median"), Map.entry("p75", "75th percentile (Q3)"), Map.entry("p95", "95th percentile"), Map.entry("p99", "99th percentile"), + Map.entry("minLength", "Shortest"), Map.entry("maxLength", "Longest"), Map.entry("meanLength", "Average length"), + Map.entry("earliest", "Earliest"), Map.entry("latest", "Latest"), Map.entry("range", "Range"), + Map.entry("true", "True"), Map.entry("false", "False"), Map.entry("other", "Other")); + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private MetricLabels() + { + } + + + + /******************************************************************************* + ** Fail at construction when a new metric lacks a human label. + *******************************************************************************/ + public static String label(String key, ValueType type) + { + if(type == ValueType.DATE || type == ValueType.DATE_TIME) + { + key = "min".equals(key) ? "earliest" : "max".equals(key) ? "latest" : key; + } + return (Objects.requireNonNull(LABELS.get(key), "Missing metric label: " + key)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java index c9a82a2..3331d96 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/MinGroupSizeGuard.java @@ -71,7 +71,7 @@ public class MinGroupSizeGuard } if(hidden > 0) { - result.add(new ValueCount().withLabel("Other (" + hidden + " groups below " + minimum + ")").withCount(count).withShare(share)); + result.add(new ValueCount().withLabel("Other (" + hidden + " groups below " + minimum + ")").withCount(count).withShare(AnalysisFormatting.ratio(share))); } return (result); } @@ -114,7 +114,7 @@ public class MinGroupSizeGuard List booleanGroups = profile.getBooleans().stream().map(m -> new ValueCount().withValue(m.getKey()).withLabel(m.getLabel()) .withCount(((Number) m.getValue()).longValue()).withShare(ProfileSupport.share(((Number) m.getValue()).longValue(), rows)).withDrill(m.getDrill())).toList(); profile.withBooleans(groups(booleanGroups).stream().map(v -> ProfileSupport.metric(v.getValue() == null ? "other" : v.getValue().toString(), v.getCount(), - com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withLabel(v.getLabel()).withDrill(v.getDrill())).toList()); + com.kingsrook.qbits.dataanalysis.result.ValueType.COUNT).withDrill(v.getDrill())).toList()); } } if(profile.getNumeric() != null) diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java new file mode 100644 index 0000000..76807f3 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/NiceBins.java @@ -0,0 +1,62 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis.engine; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.util.List; + +/******************************************************************************* + ** Exact decimal boundaries with a readable step and approximately twenty bins. + *******************************************************************************/ +public record NiceBins(BigDecimal low, BigDecimal high, int count) +{ + /******************************************************************************* + ** Select the closest 10..25-bin enclosure from 1, 2, 2.5 and 5 times powers of ten. + *******************************************************************************/ + public static NiceBins covering(BigDecimal low, BigDecimal high) + { + BigDecimal span = high.subtract(low); + if(span.signum() == 0) + { + span = low.abs().max(BigDecimal.ONE); + low = low.subtract(span.divide(BigDecimal.valueOf(2))); + high = high.add(span.divide(BigDecimal.valueOf(2))); + } + int exponent = span.precision() - span.scale() - 2; + NiceBins best = null; + for(int power = exponent - 2; power <= exponent + 2; power++) + { + for(String factor : List.of("1", "2", "2.5", "5")) + { + BigDecimal step = new BigDecimal(factor).scaleByPowerOfTen(power); + BigDecimal first = low.divide(step, 0, RoundingMode.FLOOR).multiply(step); + BigDecimal last = high.divide(step, 0, RoundingMode.CEILING).multiply(step); + int count = last.subtract(first).divide(step).intValueExact(); + if(count >= 10 && count <= 25 && (best == null || Math.abs(count - 20) < Math.abs(best.count() - 20))) + { + best = new NiceBins(first, last, count); + } + } + } + return (java.util.Objects.requireNonNull(best)); + } +} diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java index 83c8b59..716de31 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/ProfileSupport.java @@ -56,7 +56,7 @@ public final class ProfileSupport *******************************************************************************/ public static BigDecimal share(long count, long rows) { - return (rows == 0 ? BigDecimal.ZERO : BigDecimal.valueOf(count).divide(BigDecimal.valueOf(rows), MathContext.DECIMAL128)); + return (rows == 0 ? BigDecimal.ZERO : BigDecimal.valueOf(count).divide(BigDecimal.valueOf(rows), 6, java.math.RoundingMode.HALF_UP).stripTrailingZeros()); } @@ -76,7 +76,12 @@ public final class ProfileSupport *******************************************************************************/ public static Metric metric(String key, Serializable value, ValueType type) { - return (new Metric().withKey(key).withLabel(key).withValue(value).withValueType(type).withFormatted(value == null ? null : value.toString())); + if(type == ValueType.PERCENT && value != null) + { + value = AnalysisFormatting.ratio(new BigDecimal(value.toString())); + } + return (new Metric().withKey(key).withLabel(MetricLabels.label(key, type)).withValue(value).withValueType(type) + .withFormatted(AnalysisFormatting.scalar(key, value, type))); } @@ -116,7 +121,7 @@ public final class ProfileSupport label = translated; } } - result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(group.share() == null ? share(group.count(), total) : group.share()) + result.add(new ValueCount().withValue(group.value()).withLabel(label).withCount(group.count()).withShare(group.share() == null ? share(group.count(), total) : AnalysisFormatting.ratio(group.share())) .withDrill(DrillFilters.drill(table, filter, group.condition()))); } return (result); diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java index a8ee186..48b342f 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/StandInAnalysisBackend.java @@ -101,7 +101,7 @@ public class StandInAnalysisBackend implements AnalysisBackend public Scope scope(String table, QQueryFilter filter, boolean exact) throws QException { long total = count(table, filter); - boolean estimate = !exact && total > config.getSampleThresholdRows(); + boolean estimate = !exact && total > config.getSampleThresholdRows() && total > Math.min(config.getSampleRows(), config.getMaxRows()); estimating = estimate; int limit = estimate ? Math.min(config.getSampleRows(), config.getMaxRows()) : config.getMaxRows(); QQueryFilter scoped = DrillFilters.copy(filter); @@ -455,17 +455,21 @@ public class StandInAnalysisBackend implements AnalysisBackend { return (result); } + NiceBins nice = NiceBins.covering(low, high); + low = nice.low(); + high = nice.high(); + bins = nice.count(); List values = rows(table, filter, Set.of(field), config.getMaxRows()).stream().map(r -> r.getValue(field)) .filter(Objects::nonNull).map(v -> new BigDecimal(v.toString())).toList(); - result.add(bucket(table, filter, "Below " + low, numericRange(table, field, values, null, low, false))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), null, low), numericRange(table, field, values, null, low, false))); BigDecimal width = high.subtract(low).divide(BigDecimal.valueOf(bins), MATH); for(int i = 0; i < bins; i++) { BigDecimal start = low.add(width.multiply(BigDecimal.valueOf(i))); BigDecimal end = i == bins - 1 ? high : low.add(width.multiply(BigDecimal.valueOf(i + 1))); - result.add(bucket(table, filter, "[" + start + ", " + end + (i == bins - 1 ? "]" : ")"), numericRange(table, field, values, start, end, i == bins - 1))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), start, end), numericRange(table, field, values, start, end, i == bins - 1))); } - result.add(bucket(table, filter, "Above " + high, DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)))); + result.add(bucket(table, filter, AnalysisFormatting.interval(field(table, field), high, null), DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.IS_NOT_BLANK), DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN, high)))); return (result); } @@ -491,7 +495,26 @@ public class StandInAnalysisBackend implements AnalysisBackend if(unit == DateUnit.DAY_OF_WEEK || unit == DateUnit.HOUR_OF_DAY) { return (mappedGroups(table, filter, field, records, v -> unit == DateUnit.DAY_OF_WEEK ? day(v, zone).getDayOfWeek().getValue() - : ((Instant) v).atZone(zone).getHour(), 24)); + : ((Instant) v).atZone(zone).getHour(), 24).stream() + .map(g -> new GroupCount(g.value(), AnalysisFormatting.dateBucket(g.value(), unit, zone), g.count(), g.condition(), g.share())).toList()); + } + if(unit == DateUnit.HOUR) + { + List instants = records.stream().map(r -> (Instant) r.getValue(field)).filter(Objects::nonNull).sorted().toList(); + List hours = new ArrayList<>(); + if(!instants.isEmpty()) + { + Instant start = instants.getFirst().atZone(zone).truncatedTo(java.time.temporal.ChronoUnit.HOURS).toInstant(); + while(!start.isAfter(instants.getLast())) + { + Instant end = start.plus(1, java.time.temporal.ChronoUnit.HOURS); + QQueryFilter condition = DrillFilters.and(DrillFilters.condition(field, QCriteriaOperator.GREATER_THAN_OR_EQUALS, start), + DrillFilters.condition(field, QCriteriaOperator.LESS_THAN, end)); + hours.add(new GroupCount(start, AnalysisFormatting.dateBucket(start, unit, zone), count(table, DrillFilters.and(filter, condition)), condition, countShare())); + start = end; + } + } + return (hours); } List dates = records.stream().map(r -> r.getValue(field)).filter(Objects::nonNull).map(v -> day(v, zone)).sorted().toList(); List result = new ArrayList<>(); @@ -504,7 +527,8 @@ public class StandInAnalysisBackend implements AnalysisBackend while(!start.isAfter(last)) { LocalDate end = next(start, unit); - result.add(bucket(table, filter, start + " / " + end, DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone))); + QQueryFilter condition = DrillFilters.dateRange(field, start, end, field(table, field).getType() == QFieldType.DATE_TIME, zone); + result.add(new GroupCount(start, AnalysisFormatting.dateBucket(start, unit, zone), count(table, DrillFilters.and(filter, condition)), condition, countShare())); start = end; } return (result); @@ -531,6 +555,7 @@ public class StandInAnalysisBackend implements AnalysisBackend { case WEEK -> date.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)); case MONTH -> date.withDayOfMonth(1); + case QUARTER -> date.withMonth((date.getMonthValue() - 1) / 3 * 3 + 1).withDayOfMonth(1); case YEAR -> date.withDayOfYear(1); default -> date; }); @@ -547,6 +572,7 @@ public class StandInAnalysisBackend implements AnalysisBackend { case WEEK -> date.plusWeeks(1); case MONTH -> date.plusMonths(1); + case QUARTER -> date.plusMonths(3); case YEAR -> date.plusYears(1); default -> date.plusDays(1); }); @@ -605,12 +631,14 @@ public class StandInAnalysisBackend implements AnalysisBackend ** Shapes use a primary-key prefix of at most the requested sample size. *******************************************************************************/ @Override - public List patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException + public PatternSummary patterns(String table, QQueryFilter filter, String field, int sampleRows) throws QException { - List records = rows(table, filter, Set.of(field, primaryKey(table)), sampleRows); + List records = rows(table, DrillFilters.and(filter, DrillFilters.nonNull(field, true)), Set.of(field, primaryKey(table)), sampleRows + 1); + boolean truncated = records.size() > sampleRows; + records = records.subList(0, Math.min(records.size(), sampleRows)); QQueryFilter sampled = records.isEmpty() ? DrillFilters.in(primaryKey(table), List.of()) : DrillFilters.condition(primaryKey(table), QCriteriaOperator.LESS_THAN_OR_EQUALS, records.getLast().getValue(primaryKey(table))); - return (mappedGroups(table, estimating ? filter : DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10)); + return (new PatternSummary(mappedGroups(table, estimating ? filter : DrillFilters.and(filter, sampled), field, records, v -> shape(v.toString()), 10), records.size(), truncated)); } diff --git a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java index 2560174..14a8a1d 100644 --- a/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java +++ b/src/main/java/com/kingsrook/qbits/dataanalysis/engine/TableProfiler.java @@ -70,7 +70,7 @@ public class TableProfiler String name = field.getName(); AnalysisBackend.FieldCounts c = counts.get(name); result.getFields().add(new FieldSummary().withField(name).withLabel(field.getLabel()).withType(field.getType().name()) - .withCompleteness(c.completeness()).withDistinct(c.distinct()).withUniqueness(c.uniqueness()) + .withCompleteness(AnalysisFormatting.ratio(c.completeness())).withDistinct(c.distinct()).withUniqueness(AnalysisFormatting.ratio(c.uniqueness())) .withConstant(c.constant()).withEmpty(c.blanks() == rows) .withNulls(ProfileSupport.count("nulls", c.nulls(), table, drillFilter, DrillFilters.nulls(name, field.getType().isStringLike()))) .withBlanks(ProfileSupport.count("blanks", c.blanks(), table, drillFilter, c.blankFilter()))); diff --git a/src/main/resources/openapi/qqq-v1/data-analysis.yaml b/src/main/resources/openapi/qqq-v1/data-analysis.yaml index 7f0b3df..de5cdc6 100644 --- a/src/main/resources/openapi/qqq-v1/data-analysis.yaml +++ b/src/main/resources/openapi/qqq-v1/data-analysis.yaml @@ -511,7 +511,7 @@ components: nullable: true type: "string" completeness: - description: "Completeness." + description: "Non-empty share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" distinct: @@ -519,7 +519,7 @@ components: nullable: true type: "integer" uniqueness: - description: "Uniqueness." + description: "Distinct share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" constant: @@ -567,11 +567,13 @@ components: nullable: true type: "string" label: - description: "Label." + description: "Plain-English metric label, ready to display." nullable: true type: "string" value: - description: "Value." + description: "Raw numeric or temporal value. PERCENT is a 0..1 ratio rounded\ + \ to six decimals; DECIMAL field statistics have ten significant digits.\ + \ DATE_TIME is an ISO instant." nullable: true oneOf: - type: "string" @@ -596,7 +598,8 @@ components: \ remains exact." type: "boolean" formatted: - description: "Formatted." + description: "Display-ready value, including grouping, field display formats,\ + \ percent signs and session-zone dates; null when absent or suppressed." nullable: true type: "string" drill: @@ -872,7 +875,7 @@ components: nullable: true type: "integer" share: - description: "Share." + description: "Share as a 0..1 ratio rounded to six decimals." nullable: true type: "number" drill: diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java new file mode 100644 index 0000000..f454bee --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/AnalysisFormattingTest.java @@ -0,0 +1,158 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.math.BigDecimal; +import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneId; +import java.util.List; +import java.util.Map; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisFormatting; +import com.kingsrook.qbits.dataanalysis.engine.MetricLabels; +import com.kingsrook.qbits.dataanalysis.engine.NiceBins; +import com.kingsrook.qbits.dataanalysis.engine.ProfileSupport; +import com.kingsrook.qbits.dataanalysis.result.ValueType; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/******************************************************************************* + ** Public presentation contracts, including precision and calendar edge cases. + *******************************************************************************/ +class AnalysisFormattingTest +{ + /******************************************************************************* + ** Raw ratios remain numeric while display strings are ready for a CLI. + *******************************************************************************/ + @Test + void numbersPercentagesAndLengths() + { + assertThat(ProfileSupport.metric("rows", 12498L, ValueType.COUNT).getFormatted()).isEqualTo("12,498"); + for(String key : List.of("uniqueness", "completeness", "share")) + { + var metric = ProfileSupport.metric(key, new BigDecimal("0.987654321"), ValueType.PERCENT); + assertThat(metric.getValue()).isEqualTo(new BigDecimal("0.987654")); + assertThat(metric.getFormatted()).isEqualTo("98.8%"); + assertThat(metric.getValueType()).isEqualTo(ValueType.PERCENT); + } + assertThat(ProfileSupport.metric("share", BigDecimal.ONE, ValueType.PERCENT).getFormatted()).isEqualTo("100%"); + assertThat(ProfileSupport.metric("share", BigDecimal.ZERO, ValueType.PERCENT).getFormatted()).isEqualTo("0%"); + assertThat(ProfileSupport.share(1, 3)).isEqualByComparingTo("0.333333"); + assertThat(ProfileSupport.share(0, 0)).isZero(); + assertThat(ProfileSupport.metric("meanLength", new BigDecimal("12.345"), ValueType.DECIMAL).getFormatted()).isEqualTo("12.3"); + assertThat(ProfileSupport.metric("minLength", BigDecimal.TEN, ValueType.DECIMAL).getFormatted()).isEqualTo("10"); + assertThat(ProfileSupport.metric("max", null, ValueType.DECIMAL).getFormatted()).isNull(); + assertThatThrownBy(() -> MetricLabels.label("unknown", ValueType.COUNT)).isInstanceOf(NullPointerException.class).hasMessageContaining("Missing metric label"); + } + + + + /******************************************************************************* + ** Host currency and custom display formats take precedence over numeric defaults. + *******************************************************************************/ + @Test + void fieldFormatsAndRawPrecision() + { + QFieldMetaData decimal = new QFieldMetaData("amount", QFieldType.DECIMAL); + assertThat(AnalysisFormatting.numeric(decimal, "min", new BigDecimal("81753.32"))).isEqualTo("81,753.32"); + decimal.withDisplayFormat(DisplayFormat.CURRENCY); + assertThat(AnalysisFormatting.numeric(decimal, "min", new BigDecimal("81753.32"))).isEqualTo("$81,753.32"); + var metric = AnalysisFormatting.numericMetric(decimal, "mean", new BigDecimal("125916.45167830173")); + assertThat(metric.getValue()).isEqualTo(new BigDecimal("125916.4517")); + assertThat(metric.getFormatted()).isEqualTo("$125,916.45"); + assertThat(AnalysisFormatting.numericMetric(decimal, "max", null).getValue()).isNull(); + QFieldMetaData integer = new QFieldMetaData("quantity", QFieldType.INTEGER); + assertThat(AnalysisFormatting.numeric(integer, "max", new BigDecimal("12498"))).isEqualTo("12,498"); + assertThat(AnalysisFormatting.numeric(integer, "mean", new BigDecimal("12.345"))).isEqualTo("12.35"); + assertThat(AnalysisFormatting.numeric(integer, "stddev", new BigDecimal("12.345"))).isEqualTo("12.35"); + assertThat(AnalysisFormatting.numeric(integer.withDisplayFormat(DisplayFormat.DECIMAL1_COMMAS), "mean", new BigDecimal("12.345"))).isEqualTo("12.3"); + QFieldMetaData large = new QFieldMetaData("large", QFieldType.LONG).withDisplayFormat(DisplayFormat.COMMAS); + assertThat(AnalysisFormatting.numeric(large, "sum", new BigDecimal("9000000000"))).isEqualTo("9,000,000,000"); + } + + + + /******************************************************************************* + ** Nice boundaries remain exact at small scales, negative values and constants. + *******************************************************************************/ + @Test + void niceEdgesAndLabels() + { + for(List range : List.of(List.of("25317.13", "65781.79"), List.of("-0.0093", "0.0357"), List.of("42", "42"), List.of("0", "0"), List.of("-9", "905"))) + { + BigDecimal low = new BigDecimal(range.getFirst()); + BigDecimal high = new BigDecimal(range.getLast()); + NiceBins bins = NiceBins.covering(low, high); + assertThat(bins.count()).isBetween(10, 25); + assertThat(bins.low()).isLessThanOrEqualTo(low); + assertThat(bins.high()).isGreaterThanOrEqualTo(high); + BigDecimal step = bins.high().subtract(bins.low()).divide(BigDecimal.valueOf(bins.count())).stripTrailingZeros(); + assertThat(step.scaleByPowerOfTen(step.scale() - step.precision() + 1).toPlainString()).isIn("1", "2", "2.5", "5"); + assertThat(bins.low().remainder(step)).isZero(); + assertThat(bins.high().remainder(step)).isZero(); + } + QFieldMetaData field = new QFieldMetaData("amount", QFieldType.INTEGER); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("60000"), new BigDecimal("65000"))).isEqualTo("60,000 – 65,000"); + assertThat(AnalysisFormatting.interval(field, null, new BigDecimal("25000"))).isEqualTo("Below 25,000"); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("65000"), null)).isEqualTo("65,000 and above"); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("1.25"), new BigDecimal("1.5"))).isEqualTo("1.25 – 1.5"); + field.withDisplayFormat(DisplayFormat.CURRENCY); + assertThat(AnalysisFormatting.interval(field, new BigDecimal("60000"), new BigDecimal("65000"))).isEqualTo("$60,000 – $65,000"); + } + + + + /******************************************************************************* + ** Every supported unit has a human label without replacing its ISO coordinate. + *******************************************************************************/ + @Test + void dateUnitsAndDurations() + { + ZoneId zone = ZoneId.of("America/Chicago"); + LocalDate date = LocalDate.of(2024, 3, 4); + Map labels = Map.of( + AnalysisBackend.DateUnit.YEAR, "2024", AnalysisBackend.DateUnit.QUARTER, "2024 Q1", AnalysisBackend.DateUnit.MONTH, "Mar 2024", + AnalysisBackend.DateUnit.WEEK, "Week of 2024-03-04", AnalysisBackend.DateUnit.DAY, "2024-03-04"); + labels.forEach((unit, label) -> assertThat(AnalysisFormatting.dateBucket(date, unit, zone)).isEqualTo(label)); + assertThat(AnalysisFormatting.dateBucket(Instant.parse("2024-03-04T19:00:00Z"), AnalysisBackend.DateUnit.HOUR, zone)).isEqualTo("2024-03-04 13:00"); + for(int i = 1; i <= 7; i++) + { + assertThat(AnalysisFormatting.dateBucket(i, AnalysisBackend.DateUnit.DAY_OF_WEEK, zone)) + .isEqualTo(List.of("Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday").get(i - 1)); + } + for(int i = 0; i < 24; i++) + { + assertThat(AnalysisFormatting.dateBucket(i, AnalysisBackend.DateUnit.HOUR_OF_DAY, zone)).isEqualTo(String.format(java.util.Locale.US, "%02d:00", i)); + } + assertThat(AnalysisFormatting.duration(date, date.plusYears(48).plusMonths(2))).isEqualTo("48 years 2 months"); + assertThat(AnalysisFormatting.duration(date, date.plusDays(12))).isEqualTo("12 days"); + assertThat(AnalysisFormatting.duration(date, date.plusYears(1).plusMonths(1).plusDays(1))).isEqualTo("1 year 1 month 1 day"); + assertThat(AnalysisFormatting.duration(date, date)).isEqualTo("0 days"); + assertThat(ProfileSupport.metric("min", date, ValueType.DATE).getLabel()).isEqualTo("Earliest"); + assertThat(ProfileSupport.metric("max", date, ValueType.DATE).getLabel()).isEqualTo("Latest"); + assertThat(ProfileSupport.metric("min", date, ValueType.DATE).getFormatted()).isEqualTo("2024-03-04"); + } +} diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java index 324d20e..ed2a248 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/CoreAnalysisBackendTest.java @@ -110,7 +110,7 @@ class CoreAnalysisBackendTest extends EngineFixture } /******************************************************************************* - ** Zero and capped percentages never divide by zero or restrict the row drill. + ** Empty samples keep the population drill; inspecting every row is exact. *******************************************************************************/ @Test void samplingBoundaries() throws Exception @@ -123,7 +123,8 @@ class CoreAnalysisBackendTest extends EngineFixture config.withDefaultSampleRows(100); service.getConfigResolver().invalidate(); var scope = service.backend("orders").scope("orders", null, false); - assertThat(scope.samplePercent()).isEqualByComparingTo("100"); + assertThat(scope.samplePercent()).isNull(); + assertThat(scope.estimate()).isFalse(); assertThat(scope.rowsScanned()).isEqualTo(12); } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java index 68c7fd9..f8439b7 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/DataAnalysisHomeTest.java @@ -33,7 +33,6 @@ import com.kingsrook.qqq.backend.core.model.session.QSession; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; /******************************************************************************* @@ -62,8 +61,12 @@ class DataAnalysisHomeTest QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); String html = DataAnalysisHome.render(Instant.now()); assertThat(html).contains("Mine: 0", "Starred by me: 0", "No saved analyses yet", - "No active watches", "No completed or failed runs", "View all"); - assertThat(new DataAnalysisHome().render(new RenderWidgetInput())).isNotNull(); + "No watches yet.", "No analysis runs yet.", "View all", "Open any table and choose Analyze on a column header to explore its data.", "

"); + var widget = new DataAnalysisHome().render(new RenderWidgetInput()).getWidgetData(); + assertThat(widget).isInstanceOf(com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML.class); + assertThat(((com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML) widget).getType()).isEqualTo("html"); + assertThat(html).contains("·").doesNotContain("Â", "later", "phase", " c < 128)).isTrue(); QContext.init(instance, TestSupport.user("alice")); html = DataAnalysisHome.render(Instant.now()); assertThat(html).contains("you need data_analysis.saved.read", "you need data_analysis.watch.read").doesNotContain("Mine: 0", "Complete: 0"); @@ -99,11 +102,39 @@ class DataAnalysisHomeTest run("bob", "FAILED", now); QContext.init(instance, TestSupport.user("alice").withPermissions("data_analysis.saved.read", "data_analysis.watch.read", "data_analysis.run.read")); String html = DataAnalysisHome.render(now); - assertThat(html).contains("Mine: 2", "Shared (visible to you): 1", "Starred by me: 1", "Enabled: 2", "Firing: 1", "Complete: 2", "Failed: 1") - .doesNotContain("No saved analyses", "No active watches", "No completed or failed runs"); + assertThat(html).contains("Mine: 2", "Shared (visible to you): 1", "Starred by me: 1", "Enabled: 2", "Firing: 1", "Recent analysis runs", "Complete", "Failed", "Running", "Not finished") + .doesNotContain("No saved analyses", "No watches yet", "No analysis runs yet"); QContext.init(instance, TestSupport.user("bob").withPermissions("data_analysis.saved.read", "data_analysis.run.read")); html = DataAnalysisHome.render(now); - assertThat(html).contains("Mine: 1", "Shared (visible to you): 1", "Starred by me: 0", "Complete: 0", "Failed: 1", "you need data_analysis.watch.read"); + assertThat(html).contains("Mine: 1", "Shared (visible to you): 1", "Starred by me: 0", "Failed", "you need data_analysis.watch.read"); + } + + + + /******************************************************************************* + ** Latest five remain owner-scoped even with administrator bypass and escape HTML. + *******************************************************************************/ + @Test + void recentRunDetailsAndLimit() throws Exception + { + QInstance instance = TestSupport.instance(); + instance.getTable(DataAnalysisTables.SAVED).withLabel("Saved & review"); + Instant now = Instant.parse("2026-10-13T23:10:00Z"); + for(int i = 0; i < 7; i++) + { + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", DataAnalysisTables.SAVED) + .withValue("operation", "COLUMN_PROFILE").withValue("user_id", "admin").withValue("status", "COMPLETE") + .withValue("started_at", now.minusSeconds(i * 60L)).withValue("finished_at", now.minusSeconds(i * 60L))); + } + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "private_bob_table") + .withValue("operation", "TABLE_PROFILE").withValue("user_id", "bob").withValue("status", "FAILED").withValue("started_at", now)); + QContext.init(instance, TestSupport.admin().withValue(QSession.VALUE_KEY_USER_TIMEZONE, "America/Chicago")); + String html = DataAnalysisHome.render(now); + assertThat(html).contains("Saved <analysis> & review", "Column Profile", "2026-10-13 18:10", "2026-10-13 18:06", ""); + assertThat(html.split("
  • ", -1)).hasSize(6); + assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "/1\""); + assertThat(html).contains("href=\"" + instance.getTablePath(DataAnalysisTables.RUN) + "\">View all"); } @@ -113,7 +144,7 @@ class DataAnalysisHomeTest *******************************************************************************/ private static void run(String user, String status, Instant started) throws Exception { - TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "orders").withValue("operation", "profile") + TestSupport.insert(DataAnalysisTables.RUN, new QRecord().withValue("table_name", "orders").withValue("operation", "COLUMN_PROFILE") .withValue("user_id", user).withValue("status", status).withValue("started_at", started)); } } diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java index c5343e5..3275715 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineContractTest.java @@ -64,7 +64,60 @@ import static org.assertj.core.api.Assertions.within; abstract class EngineContractTest extends EngineFixture { /******************************************************************************* - ** All numeric moments and percentiles agree to 1e-9. + ** Every emitted key across all profile kinds has a label, on every backend. + *******************************************************************************/ + @Test + void presentationAcrossProfileKinds() throws Exception + { + java.util.Set keys = new java.util.HashSet<>(); + for(String field : List.of("amount", "quantity", "note", "paid", "order_date", "created")) + { + ColumnProfile profile = service.columnProfile(column(field)); + java.util.List metrics = new java.util.ArrayList<>(profile.getCounts()); + if(profile.getNumeric() != null) + { + metrics.addAll(profile.getNumeric().getMetrics()); + } + if(profile.getText() != null) + { + metrics.addAll(profile.getText().getMetrics()); + } + if(profile.getDates() != null) + { + metrics.addAll(profile.getDates().getMetrics()); + assertThat(profile.getChart().getX().getLabel()).endsWith(" (day)"); + assertThat(profile.getChart().getSeries().getFirst().getPoints().getFirst().getX().toString()).isEqualTo("2026-01-01"); + } + if(profile.getBooleans() != null) + { + metrics.addAll(profile.getBooleans()); + } + for(Metric metric : metrics) + { + keys.add(metric.getKey()); + assertThat(metric.getLabel()).isEqualTo(com.kingsrook.qbits.dataanalysis.engine.MetricLabels.label(metric.getKey(), metric.getValueType())); + assertThat(metric.getFormatted()).isNotBlank(); + } + } + assertThat(keys).containsExactlyInAnyOrder("rows", "nonNull", "nulls", "blanks", "distinct", "uniqueness", "min", "max", "mean", "sum", "stddev", + "p1", "p5", "p25", "p50", "p75", "p95", "p99", "zeros", "negatives", "outliers", "minLength", "maxLength", "meanLength", "true", "false", "range"); + ColumnProfile amount = service.columnProfile(column("amount")); + assertThat(metric(amount.getNumeric().getMetrics(), "sum").getFormatted()).isEqualTo("$1,170.00"); + assertThat(amount.getNumeric().getHistogram()).allSatisfy(v -> assertThat(v.getLabel()).contains("$")); + Metric uniqueness = metric(amount.getCounts(), "uniqueness"); + assertThat(uniqueness.getValueType().name()).isEqualTo("PERCENT"); + assertThat(uniqueness.getValue().toString()).isEqualTo("0.727273"); + assertThat(uniqueness.getFormatted()).isEqualTo("72.7%"); + assertThat(metric(service.columnProfile(column("created")).getDates().getMetrics(), "min").getFormatted()).isEqualTo("2026-01-01 06:00"); + assertThat(service.columnProfile(column("note")).getWarnings()).isEmpty(); + var table = service.tableProfile(new TableProfileRequest().withTableName("orders").withFields(List.of("note"))); + assertThat(table.getRowCount().getLabel()).isEqualTo("Rows"); + assertThat(table.getFields().getFirst().getNulls().getLabel()).isEqualTo("Nulls"); + assertThat(table.getFields().getFirst().getBlanks().getLabel()).isEqualTo("Blanks"); + } + + /******************************************************************************* + ** Decimal moments retain ten significant digits; drills retain exact membership. *******************************************************************************/ @Test void numericProfile() throws Exception @@ -80,12 +133,12 @@ abstract class EngineContractTest extends EngineFixture Map.entry("zeros", "2"), Map.entry("negatives", "1"), Map.entry("outliers", "1")); for(var entry : expected.entrySet()) { - assertMetric(result.getNumeric().getMetrics(), entry.getKey(), entry.getValue()); + assertMetric(result.getNumeric().getMetrics(), entry.getKey(), new BigDecimal(entry.getValue()).round(new java.math.MathContext(10)).toPlainString()); } double[] values = {-10, 0, 0, 10, 10, 20, 20, 30, 40, 50, 1000}; double mean = 1170.0 / 11; double variance = java.util.Arrays.stream(values).map(v -> Math.pow(v - mean, 2)).sum() / 10; - assertMetric(result.getNumeric().getMetrics(), "stddev", Double.toString(Math.sqrt(variance))); + assertMetric(result.getNumeric().getMetrics(), "stddev", BigDecimal.valueOf(Math.sqrt(variance)).round(new java.math.MathContext(10)).toPlainString()); assertThat(result.getNumeric().getHistogram()).hasSize(22); assertThat(result.getNumeric().getHistogram().stream().mapToLong(v -> v.getCount()).sum()).isEqualTo(11); assertThat(result.getNumeric().getOutliers()).singleElement().satisfies(v -> assertThat(new BigDecimal(v.getValue().toString())).isEqualByComparingTo("1000")); @@ -156,7 +209,7 @@ abstract class EngineContractTest extends EngineFixture assertThat(table.getFields()).anySatisfy(f -> { assertThat(f.getField()).isEqualTo("note"); - assertThat(f.getCompleteness().doubleValue()).isCloseTo(8.0 / 12, within(1e-9)); + assertThat(f.getCompleteness().doubleValue()).isEqualTo(0.666667); }); assertThat(table.getCandidateKeys()).anyMatch(k -> k.getFields().equals(List.of("id"))); var pairs = service.tableProfile(new TableProfileRequest().withTableName("orders").withFields(List.of("status", "quantity"))); @@ -409,7 +462,7 @@ abstract class EngineContractTest extends EngineFixture assertThat(ColumnProfiler.autoBucket(first, first.plusYears(2).plusDays(1))).isEqualTo(AnalysisBackend.DateUnit.MONTH); assertThat(ColumnProfiler.autoBucket(first, first.plusYears(10))).isEqualTo(AnalysisBackend.DateUnit.MONTH); assertThat(ColumnProfiler.autoBucket(first, first.plusYears(10).plusDays(1))).isEqualTo(AnalysisBackend.DateUnit.YEAR); - for(AnalysisBackend.DateUnit unit : List.of(AnalysisBackend.DateUnit.WEEK, AnalysisBackend.DateUnit.MONTH, AnalysisBackend.DateUnit.YEAR)) + for(AnalysisBackend.DateUnit unit : List.of(AnalysisBackend.DateUnit.WEEK, AnalysisBackend.DateUnit.MONTH, AnalysisBackend.DateUnit.QUARTER, AnalysisBackend.DateUnit.YEAR)) { var buckets = service.backend("orders").dateBuckets("orders", null, "order_date", unit, java.time.ZoneId.of("America/Chicago")); assertThat(buckets.stream().mapToLong(AnalysisBackend.GroupCount::count).sum()).isEqualTo(12); diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java index 8642f4f..8c4105a 100644 --- a/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/EngineFixture.java @@ -31,6 +31,7 @@ import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.core.instances.QInstanceEnricher; import com.kingsrook.qqq.backend.core.model.data.QRecord; import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.fields.DisplayFormat; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldMetaData; import com.kingsrook.qqq.backend.core.model.metadata.fields.QFieldType; import com.kingsrook.qqq.backend.core.model.metadata.possiblevalues.QPossibleValue; @@ -96,7 +97,7 @@ abstract class EngineFixture .withPermissionRules(DataAnalysisTables.permissionRules("orders")) .withField(new QFieldMetaData("id", QFieldType.INTEGER)).withField(new QFieldMetaData("customer", QFieldType.INTEGER).withPossibleValueSourceName("customer")) .withField(new QFieldMetaData("status", QFieldType.STRING).withPossibleValueSourceName("statusValues")) - .withField(new QFieldMetaData("amount", QFieldType.DECIMAL)).withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) + .withField(new QFieldMetaData("amount", QFieldType.DECIMAL).withDisplayFormat(DisplayFormat.CURRENCY)).withField(new QFieldMetaData("quantity", QFieldType.INTEGER)) .withField(new QFieldMetaData("order_date", QFieldType.DATE)).withField(new QFieldMetaData("created", QFieldType.DATE_TIME)) .withField(new QFieldMetaData("note", QFieldType.STRING)).withField(new QFieldMetaData("paid", QFieldType.BOOLEAN)) .withField(new QFieldMetaData("modifyDate", QFieldType.DATE_TIME).withBackendName("modify_date")) diff --git a/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java b/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java new file mode 100644 index 0000000..b4a7ec4 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/dataanalysis/PresentationSamplingTest.java @@ -0,0 +1,115 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.dataanalysis; + +import java.time.Instant; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.dataanalysis.engine.AnalysisBackend; +import com.kingsrook.qbits.dataanalysis.engine.DrillFilters; +import com.kingsrook.qbits.dataanalysis.result.ColumnProfileRequest; +import com.kingsrook.qqq.backend.core.actions.tables.InsertAction; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + +/******************************************************************************* + ** Real rows prove warning boundaries and date coordinates without Docker. + *******************************************************************************/ +class PresentationSamplingTest extends EngineFixture +{ + /******************************************************************************* + ** Null prefixes do not consume the budget; warn only after 10,000 values. + *******************************************************************************/ + @Test + void patternWarningOnlyWhenTruncated() throws Exception + { + List records = new ArrayList<>(); + for(int i = 100; i <= 10101; i++) + { + records.add(new QRecord().withValue("id", i).withValue("customer", 1).withValue("note", i == 100 ? null : "AA-11")); + } + assertThat(new InsertAction().execute(new InsertInput("orders").withRecords(records)).getRecords()).allSatisfy(r -> assertThat(r.getErrors()).isNullOrEmpty()); + var filter = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 100); + var request = new ColumnProfileRequest().withTableName("orders").withFieldName("note").withFilter(filter).withExact(true); + var truncated = service.columnProfile(request); + assertThat(truncated.getText().getSampleRows()).isEqualTo(10000); + assertThat(truncated.getWarnings()).containsExactly("Patterns are based on the first 10,000 values."); + request.withFilter(DrillFilters.and(filter, DrillFilters.condition("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, 10100))); + var all = service.columnProfile(request); + assertThat(all.getText().getSampleRows()).isEqualTo(10000); + assertThat(all.getWarnings()).isEmpty(); + } + + + + /******************************************************************************* + ** Empty samples are not empty populations; full scans are not estimates. + *******************************************************************************/ + @Test + void warningsDescribeObservedScope() throws Exception + { + config.withDefaultSampleThresholdRows(1).withDefaultSampleRows(100); + var request = new ColumnProfileRequest().withTableName("orders").withFieldName("note"); + var complete = service.columnProfile(request); + assertThat(complete.getEstimate()).isFalse(); + assertThat(complete.getWarnings()).isEmpty(); + config.withDefaultSampleRows(0); + service.getConfigResolver().invalidate(); + var emptySample = service.columnProfile(request); + assertThat(emptySample.getWarnings()).contains("No sample rows were observed; run exact for exact numbers.") + .doesNotContain("No readable records match this analysis scope.", "All values in this analysis scope are null."); + var empty = service.columnProfile(request.withExact(true).withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, -1))); + assertThat(empty.getWarnings()).containsExactly("No readable records match this analysis scope."); + var nulls = service.columnProfile(request.withFilter(DrillFilters.condition("id", QCriteriaOperator.EQUALS, 1))); + assertThat(nulls.getWarnings()).containsExactly("All values in this analysis scope are null."); + } + + + + /******************************************************************************* + ** Hour buckets cross DST in real elapsed hours and preserve ISO instants. + *******************************************************************************/ + @Test + void hourlyBucketCoordinatesAndLabels() throws Exception + { + var backend = service.backend("orders"); + ZoneId zone = ZoneId.of("America/Chicago"); + var buckets = backend.dateBuckets("orders", null, "created", AnalysisBackend.DateUnit.HOUR, zone); + assertThat(buckets.getFirst().value()).isEqualTo(Instant.parse("2026-01-01T12:00:00Z")); + assertThat(buckets.getFirst().label()).isEqualTo("2026-01-01 06:00"); + assertThat(buckets.stream().mapToLong(AnalysisBackend.GroupCount::count).sum()).isEqualTo(12); + for(var bucket : buckets) + { + assertThat(backend.count("orders", bucket.condition())).isEqualTo(bucket.count()); + } + TestSupport.insert("orders", new QRecord().withValue("id", 20).withValue("customer", 1).withValue("created", Instant.parse("2026-11-01T06:30:00Z"))); + TestSupport.insert("orders", new QRecord().withValue("id", 21).withValue("customer", 1).withValue("created", Instant.parse("2026-11-01T07:30:00Z"))); + var filter = DrillFilters.condition("id", QCriteriaOperator.GREATER_THAN_OR_EQUALS, 20); + var dst = backend.dateBuckets("orders", filter, "created", AnalysisBackend.DateUnit.HOUR, zone); + assertThat(dst).hasSize(2).allSatisfy(b -> assertThat(b.label()).isEqualTo("2026-11-01 01:00")); + assertThat(dst).extracting(AnalysisBackend.GroupCount::value).doesNotHaveDuplicates(); + assertThat(dst).allSatisfy(b -> assertThat(b.count()).isEqualTo(1)); + } +}