From 9c0e7a4afa277857ecdf4908f92b4c0dfba64d1b Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 22 Oct 2026 14:33:03 -0500 Subject: [PATCH 1/4] fix: bound My Work summary pages and parent lookups Preserve exact visible counts and due-date top ten without retaining all matching rows. Recheck authorization per page and request. Full verification passes 193 tests and coverage gates. Co-Authored-By: Codex --- docs/MY-WORK-QUERY-BOUNDS.md | 31 ++++ .../collaboration/mywork/MyWorkService.java | 89 +++++++++++- .../collaboration/MyWorkQueryBudgetTest.java | 132 ++++++++++++++++++ 3 files changed, 246 insertions(+), 6 deletions(-) create mode 100644 docs/MY-WORK-QUERY-BOUNDS.md create mode 100644 src/test/java/com/kingsrook/qbits/collaboration/MyWorkQueryBudgetTest.java diff --git a/docs/MY-WORK-QUERY-BOUNDS.md b/docs/MY-WORK-QUERY-BOUNDS.md new file mode 100644 index 0000000..0fdc50e --- /dev/null +++ b/docs/MY-WORK-QUERY-BOUNDS.md @@ -0,0 +1,31 @@ +# My Work summary query bounds + +The summary previously materialized every matching task, assignment and mention. +A regression fixture measured 601 parent lookups for 601 tasks on two parents. + +The summary now scans each category in primary-key pages of at most 200 rows. +It retains at most ten display tasks and sorts them using the existing due-date +order (earliest first, null last, then ID). Counts still include every visible +matching row, rather than silently truncating at a display limit. Assignment and +mention filters remain scoped to the signed-in user. + +Each page caches parent visibility only for that page, using the existing caller +permission and record-lock checks. Missing or revoked parents contribute neither +rows nor counts; unexpected authorization/backend failures propagate. No decision +is shared across requests. The starting maximum ID bounds each category scan so +concurrent inserts cannot extend it indefinitely. Like the earlier summary, this +is not a transactionally consistent snapshot across the three categories. + +This bounds row retention, not total work independently of data size. For N +matching rows, each category needs one high-watermark query plus approximately +ceil(N/200) page queries, and one parent lookup per distinct parent per page. +Many unique parents still require linear authorization work. Per-record heavy +field size is also outside this row-count bound. Database and browser latency, +N-node presence, saturation and production capacity remain part of #1291/#1302. + +Regression evidence: the 601-task case passes a budget of at most eight parent +queries, with no task query above 200 rows; correct top-ten ordering and revoked +access on a subsequent request are checked. Another case crosses assignment and +mention page boundaries and excludes hidden parents, other users and undelivered +mentions. These tests use the framework's real memory backend instrumentation; +they are not PostgreSQL performance or deployment-capacity measurements. diff --git a/src/main/java/com/kingsrook/qbits/collaboration/mywork/MyWorkService.java b/src/main/java/com/kingsrook/qbits/collaboration/mywork/MyWorkService.java index 6697ba5..47bc446 100644 --- a/src/main/java/com/kingsrook/qbits/collaboration/mywork/MyWorkService.java +++ b/src/main/java/com/kingsrook/qbits/collaboration/mywork/MyWorkService.java @@ -23,18 +23,19 @@ package com.kingsrook.qbits.collaboration.mywork; import java.time.Instant; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import com.kingsrook.qbits.collaboration.CollaborationRecords; import com.kingsrook.qbits.collaboration.PersonalRecords; -import com.kingsrook.qbits.collaboration.assignment.AssignmentService; import com.kingsrook.qbits.collaboration.model.CollaborationTables; import com.kingsrook.qbits.collaboration.security.ParentRecordAccess; -import com.kingsrook.qbits.collaboration.tasks.TasksService; import com.kingsrook.qqq.backend.core.exceptions.QException; import com.kingsrook.qqq.backend.core.exceptions.QNotFoundException; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QCriteriaOperator; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QFilterOrderBy; import com.kingsrook.qqq.backend.core.model.actions.tables.query.QQueryFilter; import com.kingsrook.qqq.backend.core.model.data.QRecord; @@ -43,18 +44,94 @@ import com.kingsrook.qqq.backend.core.model.data.QRecord; *******************************************************************************/ public class MyWorkService { + private static final int PAGE_SIZE = 200; + private static final Comparator TASK_ORDER = Comparator.comparing((QRecord row) -> row.getValueInstant("dueAt"), Comparator.nullsLast(Comparator.naturalOrder())) + .thenComparing(row -> row.getValueInteger("id")); + /******************************************************************************* ** Open task count and first ten, assigned record counts, and unread mentions. *******************************************************************************/ public Summary summary() throws QException { - List tasks = new TasksService().mineRows("OPEN"); + String user = PersonalRecords.currentUserId(); + List tasks = new ArrayList<>(); + long taskCount = scanVisible(CollaborationTables.TASK, new QQueryFilter() + .withCriteria("assigneeUserId", QCriteriaOperator.EQUALS, user) + .withCriteria("status", QCriteriaOperator.EQUALS, "OPEN"), row -> + { + tasks.add(row); + tasks.sort(TASK_ORDER); + if(tasks.size() > 10) + { + tasks.removeLast(); + } + }); Map assignments = new LinkedHashMap<>(); - for(QRecord row : new AssignmentService().mineRows(null, true)) + scanVisible(CollaborationTables.RECORD_ASSIGNMENT, new QQueryFilter() + .withCriteria("assigneeUserId", QCriteriaOperator.EQUALS, user), + row -> assignments.merge(row.getValueString("tableName"), 1L, Long::sum)); + long mentions = scanVisible(CollaborationTables.MENTION, new QQueryFilter() + .withCriteria("mentionedUserId", QCriteriaOperator.EQUALS, user) + .withCriteria("notifiedAt", QCriteriaOperator.IS_NOT_BLANK).withCriteria("readAt", QCriteriaOperator.IS_BLANK), row -> + { + // Only the visible count is needed for mentions. + }); + return (new Summary(taskCount, List.copyOf(tasks), assignments, mentions)); + } + + /******************************************************************************* + ** Bound retained rows and visibility decisions to one page. A starting high + ** watermark prevents concurrent inserts from extending this scan indefinitely. + ** Counts remain exact for visible rows scanned; no cross-request auth cache. + *******************************************************************************/ + private long scanVisible(String table, QQueryFilter filter, java.util.function.Consumer consumer) throws QException + { + filter.withCriteria("parentDeleted", QCriteriaOperator.EQUALS, false); + List last = CollaborationRecords.query(table, filter.clone().withOrderBy(new QFilterOrderBy("id", false)).withLimit(1)); + if(last.isEmpty()) { - assignments.merge(row.getValueString("tableName"), 1L, Long::sum); + return (0); + } + int maximum = last.getFirst().getValueInteger("id"); + Integer cursor = null; + long count = 0; + while(true) + { + QQueryFilter pageFilter = filter.clone().withCriteria("id", QCriteriaOperator.LESS_THAN_OR_EQUALS, maximum) + .withOrderBy(new QFilterOrderBy("id")).withLimit(PAGE_SIZE); + if(cursor != null) + { + pageFilter.withCriteria("id", QCriteriaOperator.GREATER_THAN, cursor); + } + List rows = CollaborationRecords.query(table, pageFilter); + Map, Boolean> parents = new HashMap<>(); + for(QRecord row : rows) + { + List parent = java.util.Arrays.asList(row.getValueString("tableName"), row.getValueString("recordId")); + if(!parents.containsKey(parent)) + { + try + { + ParentRecordAccess.requireVisibleParent(parent.get(0), parent.get(1)); + parents.put(parent, true); + } + catch(QNotFoundException expected) + { + parents.put(parent, false); + } + } + if(parents.get(parent)) + { + consumer.accept(row); + count++; + } + } + if(rows.size() < PAGE_SIZE || rows.getLast().getValueInteger("id") >= maximum) + { + return (count); + } + cursor = rows.getLast().getValueInteger("id"); } - return (new Summary((long) tasks.size(), tasks.stream().limit(10).toList(), assignments, (long) unread().size())); } /******************************************************************************* diff --git a/src/test/java/com/kingsrook/qbits/collaboration/MyWorkQueryBudgetTest.java b/src/test/java/com/kingsrook/qbits/collaboration/MyWorkQueryBudgetTest.java new file mode 100644 index 0000000..fb5bcfb --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/collaboration/MyWorkQueryBudgetTest.java @@ -0,0 +1,132 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.collaboration; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.collaboration.model.CollaborationTables; +import com.kingsrook.qbits.collaboration.mywork.MyWorkService; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.actions.tables.insert.InsertInput; +import com.kingsrook.qqq.backend.core.model.actions.tables.query.QueryInput; +import com.kingsrook.qqq.backend.core.model.data.QRecord; +import com.kingsrook.qqq.backend.core.modules.backend.implementations.memory.MemoryRecordStore; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + +/******************************************************************************* + ** Measure backend queries while preserving exact, caller-visible summary counts. + *******************************************************************************/ +class MyWorkQueryBudgetTest extends PersonalServiceFixture +{ + /******************************************************************************* + ** Counts cross page boundaries without including another user's or hidden work. + *******************************************************************************/ + @Test + void summaryPagesAssignmentsAndMentionsWithoutLeakingCounts() throws Exception + { + List parents = new ArrayList<>(); + List assignments = new ArrayList<>(); + List mentions = new ArrayList<>(); + for(int id = 1; id <= 208; id++) + { + if(id > 3) + { + parents.add(new QRecord().withValue("id", id).withValue("title", "Parent " + id).withValue("tenant", id == 208 ? "b" : "a")); + } + assignments.add(new QRecord().withValue("id", id).withValue("tableName", "document").withValue("recordId", Integer.toString(id)) + .withValue("assigneeUserId", id == 207 ? "bob" : "alice").withValue("parentDeleted", false)); + } + for(int id = 1; id <= 601; id++) + { + mentions.add(new QRecord().withValue("id", id).withValue("tableName", "document").withValue("recordId", Integer.toString(id % 3 + 1)) + .withValue("mentionedUserId", "alice").withValue("notifiedAt", Instant.now()).withValue("parentDeleted", false)); + } + mentions.add(new QRecord().withValue("id", 602).withValue("tableName", "document").withValue("recordId", "1") + .withValue("mentionedUserId", "bob").withValue("notifiedAt", Instant.now()).withValue("parentDeleted", false)); + mentions.add(new QRecord().withValue("id", 603).withValue("tableName", "document").withValue("recordId", "1") + .withValue("mentionedUserId", "alice").withValue("parentDeleted", false)); + MemoryRecordStore.getInstance().insert(new InsertInput("document").withRecords(parents), true); + MemoryRecordStore.getInstance().insert(new InsertInput(CollaborationTables.RECORD_ASSIGNMENT).withRecords(assignments), true); + MemoryRecordStore.getInstance().insert(new InsertInput(CollaborationTables.MENTION).withRecords(mentions), true); + MemoryRecordStore.resetStatistics(); + MemoryRecordStore.setCollectStatistics(true); + try + { + var summary = new MyWorkService().summary(); + assertThat(summary.assignedRecordsByTable()).containsOnlyKeys("document").containsEntry("document", 205L); + assertThat(summary.unreadMentionCount()).isEqualTo(401); + assertThat(summary.openTaskCount()).isZero(); + assertThat(summary.tasks()).isEmpty(); + MemoryRecordStore.getActionInputs().get(QueryInput.class).stream().map(input -> (QueryInput) input) + .filter(input -> input.getTableName().startsWith("collaboration_")) + .forEach(input -> assertThat(input.getFilter().getLimit()).isNotNull().isLessThanOrEqualTo(200)); + } + finally + { + MemoryRecordStore.setCollectStatistics(false); + MemoryRecordStore.resetStatistics(); + } + } + + /******************************************************************************* + ** Hundreds of tasks on one parent must not cause hundreds of parent lookups. + *******************************************************************************/ + @Test + void summaryBoundsPagesAndRepeatedParentLookups() throws Exception + { + List rows = new ArrayList<>(); + for(int id = 1; id <= 601; id++) + { + rows.add(new QRecord().withValue("id", id).withValue("tableName", "document") + .withValue("recordId", id == 601 ? "3" : "1").withValue("title", "Task " + id) + .withValue("assigneeUserId", "alice").withValue("createdByUserId", "alice") + .withValue("status", "OPEN").withValue("parentDeleted", false) + .withValue("dueAt", id % 2 == 0 ? Instant.parse("2026-10-22T00:00:00Z").plusSeconds(601 - id) : null)); + } + // Populate historical rows directly so current parent authorization is tested on read. + MemoryRecordStore.getInstance().insert(new InsertInput(CollaborationTables.TASK).withRecords(rows), true); + MemoryRecordStore.resetStatistics(); + MemoryRecordStore.setCollectStatistics(true); + try + { + var summary = new MyWorkService().summary(); + assertThat(summary.openTaskCount()).isEqualTo(600); + assertThat(summary.tasks()).extracting(row -> row.getValueInteger("id")) + .containsExactly(600, 598, 596, 594, 592, 590, 588, 586, 584, 582); + List queries = MemoryRecordStore.getActionInputs().get(QueryInput.class).stream() + .map(input -> (QueryInput) input).toList(); + assertThat(queries.stream().filter(input -> "document".equals(input.getTableName())).count()).isLessThanOrEqualTo(8); + assertThat(queries.stream().filter(input -> CollaborationTables.TASK.equals(input.getTableName()))) + .allSatisfy(input -> assertThat(input.getFilter().getLimit()).isNotNull().isLessThanOrEqualTo(200)); + // A second request must recheck current permissions rather than reuse cached decisions. + QContext.getQSession().getSecurityKeyValues().put("tenant", new ArrayList<>(List.of("b"))); + assertThat(new MyWorkService().summary().openTaskCount()).isEqualTo(1); + } + finally + { + MemoryRecordStore.setCollectStatistics(false); + MemoryRecordStore.resetStatistics(); + } + } +} -- 2.54.0 From 53e51029a2d68ee943e2474588768492ff441028 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 22 Oct 2026 14:36:10 -0500 Subject: [PATCH 2/4] test: verify My Work query budget against PostgreSQL Measure actual SQL calls with pg_stat_statements and preserve visible count, top-ten order and permission revocation. Focused real PostgreSQL fixture passes. Co-Authored-By: Codex --- docs/MY-WORK-QUERY-BOUNDS.md | 10 +- .../collaboration/MyWorkPostgresTest.java | 148 ++++++++++++++++++ 2 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 src/test/java/com/kingsrook/qbits/collaboration/MyWorkPostgresTest.java diff --git a/docs/MY-WORK-QUERY-BOUNDS.md b/docs/MY-WORK-QUERY-BOUNDS.md index 0fdc50e..418ab1f 100644 --- a/docs/MY-WORK-QUERY-BOUNDS.md +++ b/docs/MY-WORK-QUERY-BOUNDS.md @@ -27,5 +27,11 @@ Regression evidence: the 601-task case passes a budget of at most eight parent queries, with no task query above 200 rows; correct top-ten ordering and revoked access on a subsequent request are checked. Another case crosses assignment and mention page boundaries and excludes hidden parents, other users and undelivered -mentions. These tests use the framework's real memory backend instrumentation; -they are not PostgreSQL performance or deployment-capacity measurements. +mentions. These tests use the framework's real memory backend instrumentation. + +`MyWorkPostgresTest` repeats the 601-row case using PostgreSQL 17 and the generated +production schema. Independent `pg_stat_statements` observations verify exactly +five task queries and between one and eight parent queries. It also verifies the +same top ten, visible count and next-request permission revocation against SQL. +The fixture passes; these query counts are not sustained throughput or +deployment-capacity measurements. diff --git a/src/test/java/com/kingsrook/qbits/collaboration/MyWorkPostgresTest.java b/src/test/java/com/kingsrook/qbits/collaboration/MyWorkPostgresTest.java new file mode 100644 index 0000000..5229f9d --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/collaboration/MyWorkPostgresTest.java @@ -0,0 +1,148 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.collaboration; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.collaboration.mywork.MyWorkService; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.module.postgres.model.metadata.PostgreSQLBackendMetaData; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import static org.assertj.core.api.Assertions.assertThat; + +/******************************************************************************* + ** Real PostgreSQL query counts for the bounded, caller-authorized My Work scan. + *******************************************************************************/ +class MyWorkPostgresTest extends PersonalServiceFixture +{ + private static GenericContainer postgres; + + /******************************************************************************* + ** Collect SQL counts in an owned disposable database, never an application DB. + *******************************************************************************/ + @BeforeAll + static void startPostgres() + { + postgres = new GenericContainer<>("postgres:17-alpine") + .withEnv("POSTGRES_PASSWORD", "mywork-fixture-only").withExposedPorts(5432) + .withCommand("postgres", "-c", "shared_preload_libraries=pg_stat_statements") + .waitingFor(Wait.forLogMessage(".*database system is ready to accept connections.*", 2)); + postgres.start(); + } + + /******************************************************************************* + ** Stop only the database owned by this class. + *******************************************************************************/ + @AfterAll + static void stopPostgres() + { + if(postgres != null) + { + postgres.stop(); + } + } + + /******************************************************************************* + ** Reuse the real authorization fixture with PostgreSQL persistence. + *******************************************************************************/ + @Override + @BeforeEach + public void setUp() throws Exception + { + super.setUp(); + instance.getBackends().remove("memory"); + instance.addBackend(new PostgreSQLBackendMetaData().withName("memory").withHostName(postgres.getHost()) + .withPort(postgres.getMappedPort(5432)).withDatabaseName("postgres") + .withUsername("postgres").withPassword("mywork-fixture-only")); + try(Connection connection = connect(); var sql = connection.createStatement()) + { + sql.execute("CREATE EXTENSION pg_stat_statements"); + sql.execute(CollaborationSchema.postgresScript(config)); + sql.execute("CREATE TABLE document (id INTEGER PRIMARY KEY, title TEXT, tenant TEXT)"); + sql.execute("INSERT INTO document VALUES (1, 'Visible', 'a'), (3, 'Private', 'b')"); + sql.execute("INSERT INTO collaboration_task (id, table_name, record_id, title, assignee_user_id, created_by_user_id, status, parent_deleted, due_at) " + + "SELECT n, 'document', CASE WHEN n=601 THEN '3' ELSE '1' END, 'Task ' || n, 'alice', 'alice', 'OPEN', false, " + + "CASE WHEN n%2=0 THEN TIMESTAMPTZ '2026-10-22 00:00:00Z' + (601-n)*INTERVAL '1 second' ELSE NULL END FROM generate_series(1,601) n"); + sql.execute("SELECT pg_stat_statements_reset()"); + } + } + + /******************************************************************************* + ** Release owner pools before container teardown. + *******************************************************************************/ + @Override + public void tearDown() + { + ConnectionManager.closeConnectionProviders(instance); + super.tearDown(); + } + + /******************************************************************************* + ** Exact counts/order and bounded SQL parent/page queries on the real backend. + *******************************************************************************/ + @Test + void summaryPreservesVisibilityWithBoundedSqlQueries() throws Exception + { + var summary = new MyWorkService().summary(); + assertThat(summary.openTaskCount()).isEqualTo(600); + assertThat(summary.tasks()).extracting(row -> row.getValueInteger("id")) + .containsExactly(600, 598, 596, 594, 592, 590, 588, 586, 584, 582); + try(Connection connection = connect(); var sql = connection.createStatement(); + var rows = sql.executeQuery("SELECT query, calls FROM pg_stat_statements WHERE query ILIKE 'SELECT%'") ) + { + long parents = 0; + long tasks = 0; + while(rows.next()) + { + String query = rows.getString(1); + if(query.contains("\"document\"")) + { + parents += rows.getLong(2); + } + if(query.contains("\"collaboration_task\"")) + { + tasks += rows.getLong(2); + } + } + assertThat(parents).as("SQL parent queries").isBetween(1L, 8L); + assertThat(tasks).as("SQL task page queries").isEqualTo(5); + } + QContext.getQSession().getSecurityKeyValues().put("tenant", new ArrayList<>(List.of("b"))); + assertThat(new MyWorkService().summary().openTaskCount()).isEqualTo(1); + } + + /******************************************************************************* + ** A fixture-owned connection for setup and independent SQL observations. + *******************************************************************************/ + private Connection connect() throws Exception + { + return (DriverManager.getConnection("jdbc:postgresql://" + postgres.getHost() + ":" + postgres.getMappedPort(5432) + "/postgres", "postgres", "mywork-fixture-only")); + } +} -- 2.54.0 From 2ac067bea4af199486f90e2dc5212baf8910b8fb Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 22 Oct 2026 15:08:28 -0500 Subject: [PATCH 3/4] test: measure pooled My Work query workload Add a reproducible opt-in shared/distinct-parent workload with SQL and latency measurements, exact visibility assertions and bounded pool checks. Correct the fixture's unpooled connection churn to match the host provider; no runtime capacity claim. Co-Authored-By: Codex --- docs/MY-WORK-QUERY-BOUNDS.md | 38 +++++++++ .../collaboration/MyWorkPostgresTest.java | 83 +++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/docs/MY-WORK-QUERY-BOUNDS.md b/docs/MY-WORK-QUERY-BOUNDS.md index 418ab1f..4ea1765 100644 --- a/docs/MY-WORK-QUERY-BOUNDS.md +++ b/docs/MY-WORK-QUERY-BOUNDS.md @@ -35,3 +35,41 @@ five task queries and between one and eight parent queries. It also verifies the same top ten, visible count and next-request permission revocation against SQL. The fixture passes; these query counts are not sustained throughput or deployment-capacity measurements. + +## Reproducible local workload + +Run `mvn -Dtest=MyWorkPostgresTest -Dqqq.mywork.load=true test` with Java21 and +Docker. The fixture owns a disposable PostgreSQL17 container, uses the generated +schema and real authorization checks, and writes `target/mywork-load.csv`. +It never connects to an application database. The optional workload runs after +the normal query-budget regression so ordinary CI keeps the smaller test. + +Each scenario has 2,000 tasks: one uses ten shared parents, the other 2,000 distinct +parents. Every tenth parent is inaccessible. Two warmups precede ten measured +sequential summaries. Every sample verifies 1,800 visible tasks and the exact top +ten. SQL calls and server execution time are collected independently using +pg_stat_statements after warmup. Latency includes service execution and database +round trips, excluding fixture setup and assertions. + +CSV percentiles use nearest rank. With only ten samples, p95 and p99 both equal +the maximum observed sample; they are descriptive, not stable tail estimates. +SQL figures are totals for all ten samples. This is a single-JVM local workload, +not HTTP/browser latency, sustained concurrency, pool saturation, multi-node +presence or a deployment-capacity claim. Do not derive production alert thresholds +from these numbers. No wall-clock performance assertion is added to CI. + +The fixture explicitly uses the framework C3P0 connection provider, matching the +host's provider, with initial/minimum1, maximum3 and a two-second checkout timeout. +It verifies the pool stays within that cap, has no checkout timeouts and returns +all borrowed connections after each scenario. The original fixture used the +unpooled default; the expanded workload failed with a Windows socket-allocation +error after repeatedly opening connections. This was a fixture mismatch, not +proof of a Kof22 outage. The same workload completed with pooling. + +Observed local run (Java21, PostgreSQL17 in Docker on Windows, ten sequential +samples after two warmups): shared-parent p50=91.192ms, max=112.105ms; distinct-parent +p50=960.565ms, max=1100.260ms. SQL calls per summary were113 and2013 respectively. +This confirms linear distinct-parent authorization work remains despite bounded +row retention. The existing exact-count semantics still require examining every +matching task. No runtime optimization or production-capacity claim follows from +this fixture measurement alone. diff --git a/src/test/java/com/kingsrook/qbits/collaboration/MyWorkPostgresTest.java b/src/test/java/com/kingsrook/qbits/collaboration/MyWorkPostgresTest.java index 5229f9d..ae65573 100644 --- a/src/test/java/com/kingsrook/qbits/collaboration/MyWorkPostgresTest.java +++ b/src/test/java/com/kingsrook/qbits/collaboration/MyWorkPostgresTest.java @@ -21,7 +21,11 @@ package com.kingsrook.qbits.collaboration; +import java.nio.file.Files; +import java.nio.file.Path; import java.sql.Connection; +import java.util.Arrays; +import java.util.Locale; import java.sql.DriverManager; import java.util.ArrayList; import java.util.List; @@ -29,6 +33,9 @@ import com.kingsrook.qbits.collaboration.mywork.MyWorkService; import com.kingsrook.qqq.backend.core.context.QContext; import com.kingsrook.qqq.backend.module.postgres.model.metadata.PostgreSQLBackendMetaData; import com.kingsrook.qqq.backend.module.rdbms.jdbc.ConnectionManager; +import com.kingsrook.qqq.backend.module.rdbms.jdbc.C3P0PooledConnectionProvider; +import com.kingsrook.qqq.backend.module.rdbms.model.metadata.ConnectionPoolSettings; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -81,6 +88,10 @@ class MyWorkPostgresTest extends PersonalServiceFixture instance.addBackend(new PostgreSQLBackendMetaData().withName("memory").withHostName(postgres.getHost()) .withPort(postgres.getMappedPort(5432)).withDatabaseName("postgres") .withUsername("postgres").withPassword("mywork-fixture-only")); + var backend = (PostgreSQLBackendMetaData) instance.getBackend("memory"); + backend.setConnectionProvider(new QCodeReference(C3P0PooledConnectionProvider.class)); + backend.setConnectionPoolSettings(new ConnectionPoolSettings().withInitialPoolSize(1).withMinPoolSize(1) + .withMaxPoolSize(3).withAcquireIncrement(1).withCheckoutTimeoutSeconds(2)); try(Connection connection = connect(); var sql = connection.createStatement()) { sql.execute("CREATE EXTENSION pg_stat_statements"); @@ -136,6 +147,78 @@ class MyWorkPostgresTest extends PersonalServiceFixture } QContext.getQSession().getSecurityKeyValues().put("tenant", new ArrayList<>(List.of("b"))); assertThat(new MyWorkService().summary().openTaskCount()).isEqualTo(1); + if(Boolean.getBoolean("qqq.mywork.load")) + { + QContext.getQSession().getSecurityKeyValues().put("tenant", new ArrayList<>(List.of("a"))); + measureLoad(); + } + } + + /******************************************************************************* + ** Opt-in reproducible SQL work measurement, not a deployment capacity claim. + *******************************************************************************/ + private void measureLoad() throws Exception + { + final int tasks = 2000; + final int samples = 10; + List report = new ArrayList<>(); + report.add("tasks,parents,samples,p50_ms,p95_ms,p99_ms,sql_calls,sql_execution_ms"); + for(int parents : List.of(10, tasks)) + { + try(Connection connection = connect(); var sql = connection.createStatement()) + { + sql.execute("TRUNCATE collaboration_task, document"); + sql.execute("INSERT INTO document SELECT n, 'Document ' || n, CASE WHEN n%10=0 THEN 'b' ELSE 'a' END FROM generate_series(1," + parents + ") n"); + sql.execute("INSERT INTO collaboration_task (id, table_name, record_id, title, assignee_user_id, created_by_user_id, status, parent_deleted) " + + "SELECT n, 'document', (((n-1)%" + parents + ")+1)::text, 'Task ' || n, 'alice', 'alice', 'OPEN', false FROM generate_series(1," + tasks + ") n"); + } + // Warm the JVM and pool without counting these calls in the samples. + for(int warmup = 0; warmup < 2; warmup++) + { + assertThat(new MyWorkService().summary().openTaskCount()).isEqualTo(1800); + } + try(Connection connection = connect(); var sql = connection.createStatement()) + { + sql.execute("SELECT pg_stat_statements_reset()"); + } + double[] elapsed = new double[samples]; + for(int sample = 0; sample < samples; sample++) + { + long start = System.nanoTime(); + var summary = new MyWorkService().summary(); + elapsed[sample] = (System.nanoTime() - start) / 1_000_000.0; + assertThat(summary.openTaskCount()).isEqualTo(1800); + assertThat(summary.tasks()).extracting(row -> row.getValueInteger("id")) + .containsExactly(1, 2, 3, 4, 5, 6, 7, 8, 9, 11); + } + var backend = (PostgreSQLBackendMetaData) instance.getBackend("memory"); + var pool = ConnectionManager.getPoolMetrics(instance, backend); + // C3P0 returns connections asynchronously; allow a bounded check-in interval. + long deadline = System.nanoTime() + 2_000_000_000L; + while(pool.get("connectionsBusy").intValue() != 0 && System.nanoTime() < deadline) + { + Thread.sleep(10); + pool = ConnectionManager.getPoolMetrics(instance, backend); + } + assertThat(pool.get("connectionsTotal").intValue()).isBetween(1, 3); + assertThat(pool.get("connectionsBusy").intValue()).isZero(); + assertThat(pool.get("checkoutTimeouts").longValue()).isZero(); + Arrays.sort(elapsed); + try(Connection connection = connect(); var sql = connection.createStatement(); + var rows = sql.executeQuery("SELECT COALESCE(sum(calls),0), COALESCE(sum(total_exec_time),0) FROM pg_stat_statements " + + "WHERE query ILIKE 'SELECT%' AND (query LIKE '%\"document\"%' OR query LIKE '%\"collaboration_task\"%' " + + "OR query LIKE '%\"collaboration_record_assignment\"%' OR query LIKE '%\"collaboration_mention\"%')")) + { + assertThat(rows.next()).isTrue(); + report.add(String.format(Locale.ROOT, "%d,%d,%d,%.3f,%.3f,%.3f,%d,%.3f", tasks, parents, samples, + elapsed[4], elapsed[9], elapsed[9], rows.getLong(1), rows.getDouble(2))); + } + } + Path output = Path.of("target/mywork-load.csv"); + Files.createDirectories(output.getParent()); + Files.write(output, report); + System.out.println("My Work local PostgreSQL workload: " + output.toAbsolutePath()); + report.forEach(System.out::println); } /******************************************************************************* -- 2.54.0 From cdcf7ee36509c7a1fc6f22fa864209eb80a6ff34 Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 29 Oct 2026 09:56:39 -0500 Subject: [PATCH 4/4] docs(collaboration): record combined my work verification Co-Authored-By: Codex (GPT-6) --- docs/MY-WORK-QUERY-BOUNDS.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/MY-WORK-QUERY-BOUNDS.md b/docs/MY-WORK-QUERY-BOUNDS.md index 4ea1765..0c27f99 100644 --- a/docs/MY-WORK-QUERY-BOUNDS.md +++ b/docs/MY-WORK-QUERY-BOUNDS.md @@ -73,3 +73,20 @@ This confirms linear distinct-parent authorization work remains despite bounded row retention. The existing exact-count semantics still require examining every matching task. No runtime optimization or production-capacity claim follows from this fixture measurement alone. + +## Compatibility with the later attachment changes + +On October 26, an isolated source export combined attachment commit +`dc65a826ce362416d55d91136fa39225f4a7cf94` with the four My Work files from +`2ac067bea4af199486f90e2dc5212baf8910b8fb`. Both branches share base +`15cb8cbf72aa9a5e033280cf5a38384d1295fc7d`; their changed files do not overlap. +This was a source-overlay verification, not a branch merge or publication. + +Java 21 `./mvnw.cmd verify -Dqqq.mywork.load=true` passed all 209 tests with zero +failures, errors or skips, including the 29 attachment transaction tests. +Packaging and JaCoCo gates passed. The optional PostgreSQL workload retained the +same visible-count, top-ten, query-budget and pool assertions. Shared-parent +p50/max were 76.301/96.044 ms; distinct-parent p50/max were 939.896/1035.402 ms. +SQL counts remained 113 and 2013 per summary. These local measurements preserve +the limitations above; they do not qualify an integrated published artifact or +Kof22 deployment. The eventual integrated branch still needs its own CI gates. -- 2.54.0