james.maesandClaude Opus 5.5
20b8223ce8
Extension 1.2.0: compare two filters field by field with deltas and significance (two-proportion z, Welch's t, chi-square with validity rules and plain sentences), distribution shift over time with PSI, a pushed-down Pearson correlation matrix with sampled scatter and trend, and crosstabs with chi-square and Cramer's V. Pure-Java statistics checked against scipy reference values; the privacy guard, field access and drills apply as for profiles. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
287 lines
12 KiB
Java
287 lines
12 KiB
Java
/*
|
|
* QQQ - Low-code Application Framework for Engineers.
|
|
* Copyright (C) 2021-2025. Kingsrook, LLC
|
|
* 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States
|
|
* contact@kingsrook.com
|
|
* https://github.com/Kingsrook/
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* https://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package com.kingsrook.qbits.dataanalysis.stats;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.Arrays;
|
|
import java.util.List;
|
|
import java.util.Locale;
|
|
|
|
/*******************************************************************************
|
|
** Dependency-free statistical tests. All tests use two-sided 5% significance.
|
|
*******************************************************************************/
|
|
public final class Statistics
|
|
{
|
|
/*******************************************************************************
|
|
** Non-applicable results never contain NaN, infinity or an invented p-value.
|
|
*******************************************************************************/
|
|
public record Result(String test, Double statistic, Double df, Double pValue, boolean applicable, String sentence, Double cramersV)
|
|
{
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Utility class.
|
|
*******************************************************************************/
|
|
private Statistics()
|
|
{
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Explain failed assumptions without claiming a negative test result.
|
|
*******************************************************************************/
|
|
public static Result unavailable(String test, String reason)
|
|
{
|
|
return (new Result(test, null, null, null, false, "Not applicable: " + reason, null));
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** The strict comparison deliberately excludes p equal to 0.05.
|
|
*******************************************************************************/
|
|
public static String sentence(double p)
|
|
{
|
|
String formatted = p < 0.001 ? String.format(Locale.ROOT, "%.3g", p)
|
|
: java.math.BigDecimal.valueOf(p).setScale(p < 0.01 ? 3 : 2, java.math.RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
|
return ((p < 0.05 ? "A and B differ" : "No significant difference") + " (p = " + formatted + ")");
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Pooled standard error under equal proportions; require expected counts >= 5.
|
|
*******************************************************************************/
|
|
public static Result proportions(long successesA, long nA, long successesB, long nB)
|
|
{
|
|
String test = "Two-proportion z-test";
|
|
if(nA < 2 || nB < 2)
|
|
{
|
|
return (unavailable(test, "too few records"));
|
|
}
|
|
if(successesA < 0 || successesB < 0 || successesA > nA || successesB > nB)
|
|
{
|
|
return (unavailable(test, "invalid counts"));
|
|
}
|
|
double pooled = ((double) successesA + successesB) / ((double) nA + nB);
|
|
if(Math.min(Math.min(nA * pooled, nA * (1 - pooled)), Math.min(nB * pooled, nB * (1 - pooled))) < 5)
|
|
{
|
|
return (unavailable(test, "expected counts below 5"));
|
|
}
|
|
double z = ((double) successesA / nA - (double) successesB / nB) / Math.sqrt(pooled * (1 - pooled) * (1.0 / nA + 1.0 / nB));
|
|
double p = SpecialFunctions.gammaQ(0.5, z * z / 2);
|
|
return (new Result(test, z, null, p, true, sentence(p), null));
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Welch-Satterthwaite degrees of freedom from sample (ddof=1) variances.
|
|
*******************************************************************************/
|
|
public static Result welch(long nA, double meanA, double varianceA, long nB, double meanB, double varianceB)
|
|
{
|
|
String test = "Welch's t-test";
|
|
if(nA < 2 || nB < 2)
|
|
{
|
|
return (unavailable(test, "too few records"));
|
|
}
|
|
if(!Double.isFinite(meanA) || !Double.isFinite(meanB) || !Double.isFinite(varianceA) || !Double.isFinite(varianceB) || varianceA < 0 || varianceB < 0)
|
|
{
|
|
return (unavailable(test, "invalid numeric summary"));
|
|
}
|
|
double a = varianceA / nA;
|
|
double b = varianceB / nB;
|
|
if(a + b == 0)
|
|
{
|
|
return (unavailable(test, "both groups have zero variance"));
|
|
}
|
|
double t = (meanA - meanB) / Math.sqrt(a + b);
|
|
double df = (a + b) * (a + b) / (a * a / (nA - 1) + b * b / (nB - 1));
|
|
double p = SpecialFunctions.beta(df / (df + t * t), df / 2, 0.5);
|
|
return (new Result(test, t, df, p, true, sentence(p), null));
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Pearson chi-square without Yates correction; low expected categories pool
|
|
** into Other. Homogeneity preserves sample rows; independence pools both axes.
|
|
*******************************************************************************/
|
|
public static Result chiSquare(double[][] observed, boolean homogeneity)
|
|
{
|
|
String test = homogeneity ? "Chi-square homogeneity" : "Chi-square independence";
|
|
if(observed == null || observed.length < 2 || observed[0] == null || observed[0].length < 2)
|
|
{
|
|
return (unavailable(test, "a table of at least 2 by 2 is required"));
|
|
}
|
|
int columns = observed[0].length;
|
|
for(double[] row : observed)
|
|
{
|
|
if(row == null || row.length != columns || Arrays.stream(row).anyMatch(v -> !Double.isFinite(v) || v < 0))
|
|
{
|
|
return (unavailable(test, "invalid counts"));
|
|
}
|
|
if(Arrays.stream(row).sum() < 2)
|
|
{
|
|
return (unavailable(test, "too few records"));
|
|
}
|
|
}
|
|
double[][] table = poolColumns(observed);
|
|
if(!homogeneity)
|
|
{
|
|
table = transpose(poolColumns(transpose(table)));
|
|
double[][] rowFirst = poolColumns(transpose(poolColumns(transpose(observed))));
|
|
if(validExpected(rowFirst) && (!validExpected(table) || rowFirst.length * rowFirst[0].length > table.length * table[0].length))
|
|
{
|
|
table = rowFirst;
|
|
}
|
|
}
|
|
if(table.length < 2 || table[0].length < 2)
|
|
{
|
|
return (unavailable(test, "too few categories after merging expected counts below 5 into Other"));
|
|
}
|
|
double[] rows = Arrays.stream(table).mapToDouble(r -> Arrays.stream(r).sum()).toArray();
|
|
double[] cols = Arrays.stream(transpose(table)).mapToDouble(r -> Arrays.stream(r).sum()).toArray();
|
|
double total = Arrays.stream(rows).sum();
|
|
double statistic = 0;
|
|
for(int i = 0; i < rows.length; i++)
|
|
{
|
|
for(int j = 0; j < cols.length; j++)
|
|
{
|
|
double expected = rows[i] * cols[j] / total;
|
|
if(expected < 5)
|
|
{
|
|
return (unavailable(test, "expected counts below 5 after merging into Other"));
|
|
}
|
|
statistic += Math.pow(table[i][j] - expected, 2) / expected;
|
|
}
|
|
}
|
|
double df = (rows.length - 1.0) * (cols.length - 1.0);
|
|
double p = SpecialFunctions.gammaQ(df / 2, statistic / 2);
|
|
double v = Math.sqrt(statistic / (total * Math.min(rows.length - 1, cols.length - 1)));
|
|
return (new Result(test, statistic, df, p, true, sentence(p), v));
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Rare rows must not force every column into Other when row pooling alone
|
|
** would suffice. Prefer the valid pooling order retaining more categories.
|
|
*******************************************************************************/
|
|
private static boolean validExpected(double[][] table)
|
|
{
|
|
double[] rows = Arrays.stream(table).mapToDouble(r -> Arrays.stream(r).sum()).toArray();
|
|
double[] columns = Arrays.stream(transpose(table)).mapToDouble(r -> Arrays.stream(r).sum()).toArray();
|
|
return (rows.length >= 2 && columns.length >= 2
|
|
&& Arrays.stream(rows).min().orElse(0) * Arrays.stream(columns).min().orElse(0) / Arrays.stream(rows).sum() >= 5);
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Pool all categories with any expected count below five into one final column.
|
|
*******************************************************************************/
|
|
private static double[][] poolColumns(double[][] table)
|
|
{
|
|
double[] rows = Arrays.stream(table).mapToDouble(r -> Arrays.stream(r).sum()).toArray();
|
|
double total = Arrays.stream(rows).sum();
|
|
List<Integer> kept = new ArrayList<>();
|
|
List<Integer> other = new ArrayList<>();
|
|
for(int j = 0; j < table[0].length; j++)
|
|
{
|
|
double sum = 0;
|
|
for(double[] row : table)
|
|
{
|
|
sum += row[j];
|
|
}
|
|
double column = sum;
|
|
(Arrays.stream(rows).anyMatch(r -> r * column / total < 5) ? other : kept).add(j);
|
|
}
|
|
double[][] result = new double[rows.length][kept.size() + (other.isEmpty() ? 0 : 1)];
|
|
for(int i = 0; i < rows.length; i++)
|
|
{
|
|
for(int j = 0; j < kept.size(); j++)
|
|
{
|
|
result[i][j] = table[i][kept.get(j)];
|
|
}
|
|
for(int j : other)
|
|
{
|
|
result[i][kept.size()] += table[i][j];
|
|
}
|
|
}
|
|
return (result);
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Transposition lets independence apply the same pooling rule to both axes.
|
|
*******************************************************************************/
|
|
private static double[][] transpose(double[][] input)
|
|
{
|
|
double[][] result = new double[input[0].length][input.length];
|
|
for(int i = 0; i < input.length; i++)
|
|
{
|
|
for(int j = 0; j < input[i].length; j++)
|
|
{
|
|
result[j][i] = input[i][j];
|
|
}
|
|
}
|
|
return (result);
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Normalize counts to shares; floor zero shares at 1e-6 for finite PSI.
|
|
** This descriptive index is not a p-value or a significance test.
|
|
*******************************************************************************/
|
|
public static Double psi(double[] first, double[] last)
|
|
{
|
|
if(first == null || last == null || first.length == 0 || first.length != last.length
|
|
|| Arrays.stream(first).anyMatch(v -> !Double.isFinite(v) || v < 0) || Arrays.stream(last).anyMatch(v -> !Double.isFinite(v) || v < 0))
|
|
{
|
|
return (null);
|
|
}
|
|
double a = Arrays.stream(first).sum();
|
|
double b = Arrays.stream(last).sum();
|
|
if(a <= 0 || b <= 0)
|
|
{
|
|
return (null);
|
|
}
|
|
double result = 0;
|
|
for(int i = 0; i < first.length; i++)
|
|
{
|
|
double x = Math.max(1e-6, first[i] / a);
|
|
double y = Math.max(1e-6, last[i] / b);
|
|
result += (y - x) * Math.log(y / x);
|
|
}
|
|
return (result);
|
|
}
|
|
|
|
/*******************************************************************************
|
|
** Centered covariance avoids cancellation for large offsets; null on constants.
|
|
*******************************************************************************/
|
|
public static Double pearson(double[] x, double[] y)
|
|
{
|
|
if(x == null || y == null || x.length < 2 || x.length != y.length
|
|
|| Arrays.stream(x).anyMatch(v -> !Double.isFinite(v)) || Arrays.stream(y).anyMatch(v -> !Double.isFinite(v)))
|
|
{
|
|
return (null);
|
|
}
|
|
double meanX = Arrays.stream(x).average().orElseThrow();
|
|
double meanY = Arrays.stream(y).average().orElseThrow();
|
|
double xx = 0;
|
|
double yy = 0;
|
|
double xy = 0;
|
|
for(int i = 0; i < x.length; i++)
|
|
{
|
|
xx += (x[i] - meanX) * (x[i] - meanX);
|
|
yy += (y[i] - meanY) * (y[i] - meanY);
|
|
xy += (x[i] - meanX) * (y[i] - meanY);
|
|
}
|
|
return (xx == 0 || yy == 0 ? null : Math.max(-1, Math.min(1, xy / Math.sqrt(xx * yy))));
|
|
}
|
|
}
|