diff --git a/CHANGELOG.md b/CHANGELOG.md index 80df2fc..5545a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to this project are documented here. The format follows Keep - Allow manual Tested Pages inserts with a URL. Validate absolute HTTPS URLs and fill in the site, path and Manual source automatically; validate URL edits too. ### Added +- Every `web_performance_*` table and the Performance app carry help saying what the table is for and how its rows get created (`help/web-performance.yaml`, applied by `WebPerfHelp`; host help is never overridden). A Setup card at the top of the home lists tested pages, the lab runner, budgets (reference defaults or your own) and Google field data, each with its state and next step, using the runtime's own resolution. `WebPerfTablesGuardrailTest` fails when a table is unused, unexplained or lacks that help; `WebPerfDefaultsChangelogTest` runs the shipped changelog and its seeded budgets through real Liquibase. +- `reportRetentionDays` is now enforced: with a value above 0, Lighthouse Reports older than that many days are deleted after a result is stored, in batches, at most once an hour per process (runs, scores and findings stay). It was documented but nothing read it; the default 0 still keeps everything. - Record deploy and external site changes once, queue settled retests, and attribute lab runs to the observed build and CDN stage. - Evaluate enabled lab and URL CrUX budgets; retain breaches, confirmation evidence, automatic recovery and accurate open counts in the standard regression records. - Confirm statistical lab regressions with a median of three, reset baselines on Lighthouse major changes, and detect deterministic category/audit failures. diff --git a/README.md b/README.md index 99350fa..a7d7cd4 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,10 @@ Tier, Scope, Source, Strategy, Metric, Comparator and Severity use dropdowns wit | `web_performance_change_event` (Site Changes) | probes, other QBits | read-only | | `web_performance_comparison` (Before and After) | the QBit | read-only | +### Starter defaults and empty-state help + +The changelog's changeset 10 seeds 13 reference budgets (Google's "good" thresholds and the Lighthouse category bands); nothing else is seeded. The target table is filled by Sync pages from sitemap, and the lab, finding, report, field, regression, change and comparison tables are filled by the qbit as it runs (empty is a normal start). Every table's help (`help/web-performance.yaml`, shown on the table screen) says how its rows get created, and the Setup card on the home shows what still needs doing. Budgets with source RUM are not evaluated yet (no real-user source is wired); LAB and CRUX budgets are. + ## Permissions Tables and the app use `READ_WRITE_PERMISSIONS` with the base name `web_performance` (configurable), which gives `web_performance.read` and `web_performance.write`. Processes will add `web_performance.test`, `web_performance.manage` and `web_performance.admin` (design §8.1). No table stores a secret. diff --git a/UPGRADE.md b/UPGRADE.md new file mode 100644 index 0000000..92e39f8 --- /dev/null +++ b/UPGRADE.md @@ -0,0 +1,5 @@ +# Upgrading + +## Table help, Setup card and `reportRetentionDays` (unreleased) + +No DDL change, no new changeset and no host include change. The qbit now fills empty `help` slots on its app and tables (help a host set is kept) and adds a Setup widget to the Performance home. `reportRetentionDays` (default 0, keep forever) is now enforced: set it above 0 only if you want Lighthouse Reports older than that many days deleted. diff --git a/pom.xml b/pom.xml index 07efbfb..25e0703 100644 --- a/pom.xml +++ b/pom.xml @@ -82,6 +82,13 @@ 2.2.220 test + + + org.liquibase + liquibase-core + 4.33.0 + test + org.junit.jupiter junit-jupiter diff --git a/src/main/java/com/kingsrook/qbits/webperf/WebPerfQBitProducer.java b/src/main/java/com/kingsrook/qbits/webperf/WebPerfQBitProducer.java index b8f9db9..00e03a1 100644 --- a/src/main/java/com/kingsrook/qbits/webperf/WebPerfQBitProducer.java +++ b/src/main/java/com/kingsrook/qbits/webperf/WebPerfQBitProducer.java @@ -191,6 +191,7 @@ public class WebPerfQBitProducer implements QBitMetaDataProducer= REPORT_PRUNE_INTERVAL_MILLIS && lastReportPruneMillis.compareAndSet(last, now)); + } + + + + /******************************************************************************* + ** For tests: make the next reportPruneDue() true. + *******************************************************************************/ + public void allowReportPruneNow() + { + lastReportPruneMillis.set(0); + } + + + /******************************************************************************* ** The lab runner: the configured one, else PageSpeed Insights with the ** Google API key, else null (no key: lab tests are off). diff --git a/src/main/java/com/kingsrook/qbits/webperf/metadata/WebPerfHelp.java b/src/main/java/com/kingsrook/qbits/webperf/metadata/WebPerfHelp.java new file mode 100644 index 0000000..bc67486 --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/webperf/metadata/WebPerfHelp.java @@ -0,0 +1,134 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.webperf.metadata; + + +import java.io.IOException; +import java.io.InputStream; +import java.util.LinkedHashMap; +import java.util.Map; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.kingsrook.qbits.webperf.WebPerfQBitConfig; +import com.kingsrook.qqq.backend.core.exceptions.QException; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; +import com.kingsrook.qqq.backend.core.model.metadata.layout.QAppMetaData; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; + + +/******************************************************************************* + ** Applies help/web-performance.yaml (the QQQ help-bundle format) to the + ** Performance app and its web_performance_* tables: what each table is for and + ** how its rows get created, shown on the table screen and when it is empty. + ** STAND-IN(core=QRun-IO/qqq#1198, removeBy=4.1.0-RC.9): replace with HelpBundles.register once the pinned QQQ + ** release includes it; the bundle file stays the same. + *******************************************************************************/ +public final class WebPerfHelp +{ + public static final String RESOURCE = "/help/web-performance.yaml"; + public static final String SLOT = "help"; + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private WebPerfHelp() + { + } + + + + /******************************************************************************* + ** The bundle's entries by key, validated (fails startup if malformed). + *******************************************************************************/ + public static Map entries() throws QException + { + try(InputStream stream = WebPerfHelp.class.getResourceAsStream(RESOURCE)) + { + if(stream == null) + { + throw (new IOException("missing resource")); + } + JsonNode bundle = new ObjectMapper(new YAMLFactory()).readTree(stream); + if(!"web-performance".equals(bundle.path("bundle").asText()) || !"QBIT".equals(bundle.path("layer").asText())) + { + throw (new IOException("invalid bundle header")); + } + Map entries = new LinkedHashMap<>(); + for(JsonNode entry : bundle.path("entries")) + { + String key = entry.path("key").asText(); + String summary = entry.path("summary").asText(); + if(key.isBlank() || summary.isBlank() || summary.length() > 120 || entry.path("content").asText().isBlank() || entries.put(key, entry) != null) + { + throw (new IOException("invalid or duplicate entry: " + key)); + } + } + return (entries); + } + catch(IOException | RuntimeException e) + { + throw (new QException("Could not load " + RESOURCE, e)); + } + } + + + + /******************************************************************************* + ** Apply the help to the app and every web_performance_* table in the instance + ** (names carry the config's prefix, if any). A host's own help on a table + ** is kept (the bundle only fills the slot). + *******************************************************************************/ + public static void apply(QInstance qInstance, WebPerfQBitConfig config) throws QException + { + for(Map.Entry entry : entries().entrySet()) + { + String key = entry.getKey(); + String name = key.substring(key.indexOf(':') + 1, key.indexOf(';')); + if(key.startsWith("table:")) + { + QTableMetaData table = qInstance.getTable(config.applyPrefix(name.startsWith(WebPerfQBitConfig.DEFAULT_TABLE_PREFIX) ? name.substring(WebPerfQBitConfig.DEFAULT_TABLE_PREFIX.length()) : name)); + if(table != null && (table.getHelpContent() == null || !table.getHelpContent().containsKey(SLOT))) + { + table.withHelpContent(SLOT, content(entry.getValue())); + } + } + else if(key.startsWith("app:")) + { + QAppMetaData app = qInstance.getApp(config.getAppName()); + if(app != null && (app.getHelpContent() == null || !app.getHelpContent().containsKey(SLOT))) + { + app.withHelpContent(SLOT, content(entry.getValue())); + } + } + } + } + + + + /******************************************************************************* + ** Summary as the first paragraph, then the content, as markdown. + *******************************************************************************/ + static QHelpContent content(JsonNode entry) + { + return (new QHelpContent().withContentAsMarkdown(entry.path("summary").asText().strip() + "\n\n" + entry.path("content").asText().strip())); + } + +} diff --git a/src/main/java/com/kingsrook/qbits/webperf/metadata/WebPerfMetaData.java b/src/main/java/com/kingsrook/qbits/webperf/metadata/WebPerfMetaData.java index 76e4ae0..836c663 100644 --- a/src/main/java/com/kingsrook/qbits/webperf/metadata/WebPerfMetaData.java +++ b/src/main/java/com/kingsrook/qbits/webperf/metadata/WebPerfMetaData.java @@ -129,6 +129,7 @@ public final class WebPerfMetaData { qInstance.addProcess(process); } + addWidget(qInstance, WebPerfWidgets.SETUP, "Setup", WidgetType.HTML, WebPerfWidgets.Setup.class); addWidget(qInstance, WebPerfWidgets.SITE_SCORES, "Site Scores", WidgetType.HTML, WebPerfWidgets.SiteScores.class); addWidget(qInstance, WebPerfWidgets.TREND, "Performance Trend", WidgetType.LINE_CHART, WebPerfWidgets.Trend.class); addWidget(qInstance, WebPerfWidgets.SLOWEST_PAGES, "Slowest Pages", WidgetType.HTML, WebPerfWidgets.SlowestPages.class); diff --git a/src/main/java/com/kingsrook/qbits/webperf/service/WebPerfService.java b/src/main/java/com/kingsrook/qbits/webperf/service/WebPerfService.java index 08e3786..8aa0508 100644 --- a/src/main/java/com/kingsrook/qbits/webperf/service/WebPerfService.java +++ b/src/main/java/com/kingsrook/qbits/webperf/service/WebPerfService.java @@ -76,6 +76,9 @@ public class WebPerfService /** Statuses that still hold a place in the queue (a new request merges into them). */ public static final List OPEN_STATUSES = List.of("QUEUED", "RUNNING"); + static final int REPORT_BATCH = 500; + static final int REPORT_MAX_BATCHES = 20; + private final WebPerfRuntime runtime; private final WebPerfStore store; private final WebPerfQBitConfig config; @@ -608,6 +611,45 @@ public class WebPerfService + /******************************************************************************* + ** Apply reportRetentionDays: delete Lighthouse Reports older than that many + ** days (0 keeps them forever). The run, its scores and findings stay; only + ** the full compressed response goes. Runs after a result is committed (a + ** failure here never loses a run's evidence), at most once an hour per + ** process (WebPerfRuntime.reportPruneDue, shared by the workers), in batches + ** of REPORT_BATCH rows, at most REPORT_MAX_BATCHES per pass. There is no + ** table for state shared across replicas, so each replica keeps its own + ** hourly limit; passes are bounded and idempotent, so overlapping replicas + ** only repeat one cheap query. + *******************************************************************************/ + void pruneReports() + { + int days = runtime.getConfig().getReportRetentionDays(); + if(days <= 0 || !runtime.reportPruneDue()) + { + return; + } + + try + { + java.sql.Timestamp cutoff = WebPerfStore.utc(Instant.now().minus(java.time.Duration.ofDays(days))); + for(int batch = 0; batch < REPORT_MAX_BATCHES; batch++) + { + int deleted = store.update("DELETE FROM {lab_report} WHERE id IN (SELECT id FROM {lab_report} WHERE create_date < ? ORDER BY id FETCH FIRST " + REPORT_BATCH + " ROWS ONLY)", cutoff); + if(deleted < REPORT_BATCH) + { + return; + } + } + } + catch(SQLException | RuntimeException e) + { + LOG.warn("Could not prune old Lighthouse Reports", e); + } + } + + + /******************************************************************************* ** Store a successful response: the run's scores and metrics, its findings, ** the compressed report, and the page's latest results. @@ -619,6 +661,7 @@ public class WebPerfService new WebPerfService(runtime, tx).storeResult(runId, run, attempts, parsed, response); return (null); }); + pruneReports(); } diff --git a/src/main/java/com/kingsrook/qbits/webperf/widgets/WebPerfSetupStatus.java b/src/main/java/com/kingsrook/qbits/webperf/widgets/WebPerfSetupStatus.java new file mode 100644 index 0000000..592f26e --- /dev/null +++ b/src/main/java/com/kingsrook/qbits/webperf/widgets/WebPerfSetupStatus.java @@ -0,0 +1,183 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.webperf.widgets; + + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import com.kingsrook.qbits.webperf.WebPerfFeature; +import com.kingsrook.qbits.webperf.WebPerfQBitConfig; +import com.kingsrook.qbits.webperf.WebPerfRuntime; +import com.kingsrook.qbits.webperf.WebPerfSchema; +import com.kingsrook.qbits.webperf.store.WebPerfStore; + + +/******************************************************************************* + ** What is set up on the Performance home and what is next: whether pages have + ** been synced, whether budgets exist (and whether they are only the seeded + ** defaults), and whether the lab runner and Google field data are configured. + ** The checks use the runtime's own resolution (WebPerfRuntime.labRunner() and + ** cruxClient(), the url source), so the card never says "needs setup" for + ** something the runtime would in fact run. + *******************************************************************************/ +public final class WebPerfSetupStatus +{ + public static final String SET_UP = "Set up"; + public static final String NEEDS_SETUP = "Needs setup"; + public static final String DEFAULTS = "Using defaults"; + public static final String OPTIONAL = "Optional"; + public static final String OFF = "Off"; + + + + /******************************************************************************* + ** One line of the card. + ** + ** @param name what is being set up + ** @param state one of the state constants + ** @param detail what the state is (counts, and the next step) + ** @param tableName the table to open (prefixed name), or null + *******************************************************************************/ + public record Item(String name, String state, String detail, String tableName) + { + } + + + + /******************************************************************************* + ** Utility class. + *******************************************************************************/ + private WebPerfSetupStatus() + { + } + + + + /******************************************************************************* + ** The card's lines, in setup order. + *******************************************************************************/ + public static List items(WebPerfRuntime runtime) throws SQLException + { + WebPerfQBitConfig config = runtime.getConfig(); + WebPerfStore store = runtime.store(); + List items = new ArrayList<>(); + + ////////////////////////////////////////////////////////////////////////// + // tested pages: synced from the url source (sitemap) or added by hand // + ////////////////////////////////////////////////////////////////////////// + long pages = count(store, "SELECT COUNT(*) AS n FROM {target}"); + String target = config.applyPrefix(WebPerfSchema.TARGET); + if(pages > 0) + { + items.add(new Item("Tested pages", SET_UP, pages + " page" + (pages == 1 ? "" : "s") + " are tracked.", target)); + } + else if(config.getUrlSource() == null) + { + items.add(new Item("Tested pages", NEEDS_SETUP, "No pages yet and no page source is configured. Add a page by hand, or have the host set a URL source (the sitemap) in the qbit config.", target)); + } + else + { + items.add(new Item("Tested pages", NEEDS_SETUP, "No pages yet. Run “Sync pages from sitemap” on Tested Pages.", target)); + } + + ////////////////////////////////////////////////////////////////////////// + // lab runner: the configured one, else PageSpeed Insights with a key // + ////////////////////////////////////////////////////////////////////////// + if(!config.isOn(WebPerfFeature.LAB)) + { + items.add(new Item("Lab tests (Lighthouse)", OFF, "The LAB feature is turned off in the qbit config.", null)); + } + else if(runtime.labRunner() != null) + { + items.add(new Item("Lab tests (Lighthouse)", SET_UP, "A lab runner is configured. Use “Run site audit” on Tested Pages, or wait for the schedule.", config.applyPrefix(WebPerfSchema.LAB_RUN))); + } + else + { + items.add(new Item("Lab tests (Lighthouse)", NEEDS_SETUP, "No Google API key is configured, so lab tests cannot run. The host sets googleApiKey in the qbit config (read from an environment variable).", null)); + } + + ////////////////////////////////////////////////////////////////////////// + // budgets: the seeded reference budgets, or the admin's own // + ////////////////////////////////////////////////////////////////////////// + long budgets = count(store, "SELECT COUNT(*) AS n FROM {budget} WHERE enabled = TRUE"); + long custom = count(store, "SELECT COUNT(*) AS n FROM {budget} WHERE enabled = TRUE AND (seeded IS NULL OR seeded = FALSE)"); + String budget = config.applyPrefix(WebPerfSchema.BUDGET); + if(budgets == 0) + { + items.add(new Item("Budgets", NEEDS_SETUP, "No budget is enabled, so nothing is flagged as too slow. Add one, or enable the reference budgets.", budget)); + } + else if(custom == 0) + { + items.add(new Item("Budgets", DEFAULTS, budgets + " reference budgets (Google's “good” thresholds) are enabled. Edit them or add your own.", budget)); + } + else + { + items.add(new Item("Budgets", SET_UP, budgets + " budgets are enabled, " + custom + " of them your own.", budget)); + } + + ////////////////////////////////////////////////////////////////////////// + // Google field data (CrUX) // + ////////////////////////////////////////////////////////////////////////// + if(!config.isOn(WebPerfFeature.FIELD_CRUX)) + { + items.add(new Item("Google field data", OFF, "The FIELD_CRUX feature is turned off in the qbit config.", null)); + } + else if(runtime.cruxClient() != null) + { + items.add(new Item("Google field data", SET_UP, "Checked daily; use “Refresh Google field data” to check now.", config.applyPrefix(WebPerfSchema.FIELD_SNAPSHOT))); + } + else + { + items.add(new Item("Google field data", NEEDS_SETUP, "No Google API key is configured, so Google field data is not checked.", null)); + } + + items.add(new Item("Regressions, changes, comparisons", OPTIONAL, "Filled in by the qbit as tests run and deploys are detected. Nothing to set up.", config.applyPrefix(WebPerfSchema.REGRESSION))); + return (items); + } + + + + /******************************************************************************* + ** The card's HTML. + *******************************************************************************/ + public static String html(WebPerfRuntime runtime) throws SQLException + { + List> rows = new ArrayList<>(); + for(Item item : items(runtime)) + { + String color = NEEDS_SETUP.equals(item.state()) ? Html.POOR : SET_UP.equals(item.state()) ? Html.GOOD : Html.MEDIUM; + rows.add(List.of( + item.tableName() == null ? Html.escape(item.name()) : Html.link("/app/" + item.tableName(), item.name()), + "" + Html.escape(item.state()) + "", + Html.escape(item.detail()))); + } + return (Html.table(List.of("Part", "State", "What is next"), rows)); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static long count(WebPerfStore store, String sql) throws SQLException + { + Long n = WebPerfStore.asLong(store.query(sql).get(0).get("n")); + return (n == null ? 0 : n); + } + +} diff --git a/src/main/java/com/kingsrook/qbits/webperf/widgets/WebPerfWidgets.java b/src/main/java/com/kingsrook/qbits/webperf/widgets/WebPerfWidgets.java index 6275ffd..e648096 100644 --- a/src/main/java/com/kingsrook/qbits/webperf/widgets/WebPerfWidgets.java +++ b/src/main/java/com/kingsrook/qbits/webperf/widgets/WebPerfWidgets.java @@ -58,6 +58,7 @@ import com.kingsrook.qqq.backend.core.model.dashboard.widgets.RawHTML; *******************************************************************************/ public final class WebPerfWidgets { + public static final String SETUP = "Setup"; public static final String SITE_SCORES = "SiteScores"; public static final String TREND = "Trend"; public static final String SLOWEST_PAGES = "SlowestPages"; @@ -67,7 +68,7 @@ public final class WebPerfWidgets public static final String RECORD_PANEL = "RecordPanel"; /** The home widgets, in order. */ - public static final List HOME = List.of(SITE_SCORES, TREND, SLOWEST_PAGES, OPPORTUNITIES, FIELD_STATUS, RECENT_RUNS); + public static final List HOME = List.of(SETUP, SITE_SCORES, TREND, SLOWEST_PAGES, OPPORTUNITIES, FIELD_STATUS, RECENT_RUNS); static final int TREND_WEEKS = 26; @@ -415,6 +416,31 @@ public final class WebPerfWidgets + /******************************************************************************* + ** Setup: what is configured (pages, lab runner, budgets, field data) and + ** the next step for each (WebPerfSetupStatus). + *******************************************************************************/ + public static class Setup extends AbstractWidgetRenderer + { + /******************************************************************************* + ** Render. + *******************************************************************************/ + @Override + public RenderWidgetOutput render(RenderWidgetInput input) throws QException + { + try + { + return (html(WebPerfSetupStatus.html(runtime(input)))); + } + catch(SQLException e) + { + throw (new QException("Error loading setup status", e)); + } + } + } + + + /******************************************************************************* ** Field Data Status: what Google's field data (CrUX) says about the site, ** honestly NO_DATA when Google has none. diff --git a/src/main/resources/help/web-performance.yaml b/src/main/resources/help/web-performance.yaml new file mode 100644 index 0000000..6326daf --- /dev/null +++ b/src/main/resources/help/web-performance.yaml @@ -0,0 +1,58 @@ +bundle: web-performance +version: 1 +layer: QBIT +# Help for the Performance app and every web_performance_* table, in the QQQ +# help-bundle format (qqq#1198). Each table entry says what the table is for and +# how its rows get created, so an empty table explains itself. WebPerfHelp +# applies it until the core HelpBundles registry ships in a QQQ release. +# Summaries stay under 120 characters (WebPerfTablesGuardrailTest). +entries: + - key: app:webPerformance;slot:help + summary: "Page speed for your site: Lighthouse lab tests, Google field data, budgets and regressions." + content: | + The Setup card on the home lists what is configured and the next step: sync the pages, give the qbit a Google API key, and review the budgets. A new installation starts with reference budgets (Google's good thresholds) already in place. + + - key: table:web_performance_target;slot:help + summary: "The pages that get tested: one row per URL, with its tier, latest scores and open regressions." + content: | + **How rows get created:** run Sync pages from sitemap (also scheduled), which adds every page the host's URL source returns and links it to its content record. You can also add a page by hand with New. Mark a page KEY to put it in the site scores and the frequent tests. Run site audit or Test page speed queues lab tests. + + - key: table:web_performance_lab_run;slot:help + summary: "Every Lighthouse test of a page: queued, running, finished or failed, with its scores and timings." + content: | + **How rows get created:** by the qbit. Test page speed, Run site audit and the schedules queue runs here, and the worker runs them through the lab runner (PageSpeed Insights when a Google API key is set). Nothing is added by hand. Without a key, runs wait in the queue. Retry re-queues a failed run. + + - key: table:web_performance_lab_finding;slot:help + summary: "What Lighthouse says to fix on each tested page: the audits that failed and how much they would save." + content: | + **How rows get created:** by the qbit, from each finished lab run. One row per failed or opportunity audit. It stays empty until a lab test succeeds. + + - key: table:web_performance_lab_report;slot:help + summary: "The complete Lighthouse response of each run, compressed, for download and later comparison." + content: | + **How rows get created:** by the qbit, one per successful lab run. Nothing to set up. Reports are kept forever unless the host sets reportRetentionDays, which deletes older reports (the run and its scores stay). + + - key: table:web_performance_field_snapshot;slot:help + summary: "What Google's field data (CrUX) says real Chrome users experience: p75 Core Web Vitals per URL or origin." + content: | + **How rows get created:** by the qbit. Refresh Google field data (and the daily schedule) asks Google for the site and each tested page. It needs the Google API key. A row says NO_DATA when Google has too little traffic to publish figures for that page. + + - key: table:web_performance_budget;slot:help + summary: "Limits a metric must stay within, per site, tier, page, content table or path; a breach raises a regression." + content: | + **How rows get created:** the qbit seeds reference budgets (Google's good thresholds for each Core Web Vital, and Lighthouse category scores for KEY pages), so a new installation already flags slow pages. Edit them, turn them off, or add your own with New. Lab and Google field data (CRUX) budgets are evaluated; budgets with source RUM (real users) are not evaluated yet, because no real-user source is wired in. + + - key: table:web_performance_regression;slot:help + summary: "A page that got slower or broke a budget: suspected, confirmed, then resolved or dismissed." + content: | + **How rows get created:** by the qbit. When a test breaks a budget or drifts from its baseline, it opens a suspected regression and confirms it after more runs. Mark resolved and Dismiss close one by hand. Empty means nothing is wrong. + + - key: table:web_performance_change_event;slot:help + summary: "Site changes (a deploy, a Lighthouse version, a CDN stage change) that trigger re-tests." + content: | + **How rows get created:** by the qbit. Check site changes (scheduled when deploy probes or change sources are configured) records a change, and the qbit notes when Lighthouse itself changes version. It stays empty until the host configures a deploy probe or change event source. + + - key: table:web_performance_comparison;slot:help + summary: "Before and after results around a site change, per page and metric." + content: | + **How rows get created:** run Compare before and after on a change event (it needs the CDN_COMPARE feature and a change event). Nothing to add by hand. diff --git a/src/test/java/com/kingsrook/qbits/webperf/WebPerfDefaultsChangelogTest.java b/src/test/java/com/kingsrook/qbits/webperf/WebPerfDefaultsChangelogTest.java new file mode 100644 index 0000000..7458fa7 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/webperf/WebPerfDefaultsChangelogTest.java @@ -0,0 +1,139 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.webperf; + + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.Statement; +import liquibase.Liquibase; +import liquibase.database.Database; +import liquibase.database.DatabaseFactory; +import liquibase.database.jvm.JdbcConnection; +import liquibase.resource.ClassLoaderResourceAccessor; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** The shipped changelog, run through real Liquibase on H2 (PostgreSQL mode): + ** it creates the nine tables, changeset 10 seeds the reference budgets, a + ** second update runs nothing, and the seed statement run again inserts nothing + ** and keeps an administrator's edit. + *******************************************************************************/ +class WebPerfDefaultsChangelogTest +{ + private static final String PREFIX = "web_performance_"; + + + + /******************************************************************************* + ** + *******************************************************************************/ + @Test + void seededBudgetsThroughRealLiquibase_idempotent_andKeepEdits() throws Exception + { + try(Connection connection = connect()) + { + update(); + assertThat(count(connection, "SELECT COUNT(*) FROM " + PREFIX + "budget")).isEqualTo(13); + assertThat(count(connection, "SELECT COUNT(*) FROM " + PREFIX + "budget WHERE seeded = TRUE AND enabled = TRUE")).isEqualTo(13); + for(String base : WebPerfSchema.TABLES) + { + assertThat(count(connection, "SELECT COUNT(*) FROM " + PREFIX + base)).as(base + " is queryable").isGreaterThanOrEqualTo(0); + } + int changesets = count(connection, "SELECT COUNT(*) FROM DATABASECHANGELOG"); + + //////////////////////////////////////////////////////////////// + // a second update has nothing to run // + //////////////////////////////////////////////////////////////// + update(); + assertThat(count(connection, "SELECT COUNT(*) FROM DATABASECHANGELOG")).isEqualTo(changesets); + assertThat(count(connection, "SELECT COUNT(*) FROM " + PREFIX + "budget")).isEqualTo(13); + + //////////////////////////////////////////////////////////////////////////// + // an admin edits a seed; the seed statement (changeset 10) is run again: // + // nothing is inserted and the edit survives // + //////////////////////////////////////////////////////////////////////////// + try(Statement statement = connection.createStatement()) + { + statement.execute("UPDATE " + PREFIX + "budget SET threshold = 3000 WHERE name = 'LCP good (lab, mobile)'"); + statement.execute(seedStatement()); + } + assertThat(count(connection, "SELECT COUNT(*) FROM " + PREFIX + "budget")).isEqualTo(13); + assertThat(count(connection, "SELECT COUNT(*) FROM " + PREFIX + "budget WHERE name = 'LCP good (lab, mobile)' AND threshold = 3000")).isEqualTo(1); + } + } + + + + /******************************************************************************* + ** The INSERT of changeset 10, with the prefix filled in. + *******************************************************************************/ + private static String seedStatement() throws Exception + { + String script; + try(java.io.InputStream in = WebPerfDefaultsChangelogTest.class.getClassLoader().getResourceAsStream(WebPerfSchema.CHANGELOG)) + { + script = new String(in.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8); + } + String seed = script.substring(script.indexOf("-- changeset qbit-web-performance:10"), script.indexOf("-- changeset qbit-web-performance:11")); + seed = seed.substring(seed.indexOf("INSERT INTO")).trim().replaceAll(";$", ""); + return (seed.replace("${" + WebPerfSchema.CHANGELOG_PREFIX_PROPERTY + "}", PREFIX)); + } + + + + /******************************************************************************* + ** A connection to the shared in-memory database. + *******************************************************************************/ + private static Connection connect() throws Exception + { + return (DriverManager.getConnection("jdbc:h2:mem:wpdefaults;MODE=PostgreSQL;DB_CLOSE_DELAY=-1", "sa", "")); + } + + + + /******************************************************************************* + ** One real Liquibase update on its own connection (Liquibase closes it). + *******************************************************************************/ + private static void update() throws Exception + { + Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(new JdbcConnection(connect())); + try(Liquibase liquibase = new Liquibase(WebPerfSchema.CHANGELOG, new ClassLoaderResourceAccessor(), database)) + { + liquibase.setChangeLogParameter(WebPerfSchema.CHANGELOG_PREFIX_PROPERTY, PREFIX); + liquibase.update(""); + } + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static int count(Connection connection, String sql) throws Exception + { + try(Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sql)) + { + resultSet.next(); + return (resultSet.getInt(1)); + } + } + +} diff --git a/src/test/java/com/kingsrook/qbits/webperf/WebPerfP1Test.java b/src/test/java/com/kingsrook/qbits/webperf/WebPerfP1Test.java index ab503a0..ddef6cb 100644 --- a/src/test/java/com/kingsrook/qbits/webperf/WebPerfP1Test.java +++ b/src/test/java/com/kingsrook/qbits/webperf/WebPerfP1Test.java @@ -206,6 +206,67 @@ class WebPerfP1Test + /******************************************************************************* + ** The Setup card reports pages, budgets, the lab runner and field data from + ** the runtime's own resolution, with the next step for each. + *******************************************************************************/ + @Test + void testSetupCard() throws Exception + { + String card = html(WebPerfWidgets.SETUP, Map.of()); + assertThat(card).contains("Needs setup").contains("Sync pages from sitemap").contains("Using defaults").contains("13 reference budgets"); + assertThat(card).as("a configured runner and CrUX client count as set up").contains("A lab runner is configured").contains("Checked daily"); + + WebPerfService service = service(); + service.applySync(service.planSync(List.of(SITE + "/")), Instant.now()); + store().update("INSERT INTO {budget} (name, scope, source, metric, comparator, threshold, severity, enabled, seeded) VALUES ('mine', 'SITE', 'LAB', 'LCP', '<=', 2000, 'WARNING', TRUE, FALSE)"); + card = html(WebPerfWidgets.SETUP, Map.of()); + assertThat(card).contains("1 page").contains("14 budgets are enabled, 1 of them your own"); + + store().update("UPDATE {budget} SET enabled = FALSE"); + assertThat(html(WebPerfWidgets.SETUP, Map.of())).contains("No budget is enabled"); + + config.withLabRunner(null).withCruxClient(null); + assertThat(html(WebPerfWidgets.SETUP, Map.of())).contains("No Google API key is configured, so lab tests cannot run").contains("Google field data is not checked"); + } + + + + /******************************************************************************* + ** reportRetentionDays deletes old Lighthouse Reports when a run stores its + ** own, and 0 (the default) keeps them forever. + *******************************************************************************/ + @Test + void testReportRetention() throws Exception + { + for(int days : new int[] { 0, 30 }) + { + store().update("DELETE FROM {lab_report}"); + store().update("DELETE FROM {lab_finding}"); + store().update("DELETE FROM {lab_run}"); + store().update("DELETE FROM {budget}"); + config.withReportRetentionDays(days); + google.respond(new FixtureServer.Response(200, FixtureServer.fixture("psi-kof22-home-mobile.json"))); + store().insert("lab_report", new java.util.LinkedHashMap<>(Map.of("run_id", 999_999L, "bytes", 1, + "create_date", Instant.now().minus(java.time.Duration.ofDays(40))))); + WebPerfRuntime.of(qInstance, "web_performance_").allowReportPruneNow(); + WebPerfService service = service(); + service.applySync(service.planSync(List.of(SITE + "/")), Instant.now()); + service.enqueue(service.requestsFor(service.enabledTargets(false), "MANUAL", "test", Instant.now()), Instant.now()); + assertEquals(2, service.runDue(Instant.now())); + + assertEquals(2, store().query("SELECT id FROM {lab_run} WHERE status = 'SUCCEEDED'").size(), "runs are untouched by pruning"); + assertThat(store().query("SELECT id FROM {lab_finding}")).as("findings are untouched by pruning").hasSizeGreaterThan(5); + long old = store().query("SELECT id FROM {lab_report} WHERE run_id = 999999").size(); + long fresh = store().query("SELECT id FROM {lab_report} WHERE run_id <> 999999").size(); + assertEquals(2, fresh, "the new reports are kept"); + assertEquals(days == 0 ? 1 : 0, old, "retention " + days + " days"); + } + assertThat(new WebPerfRuntime(config).reportPruneDue()).as("a new process does not prune right away").isFalse(); + } + + + /******************************************************************************* ** The worker runs a test, stores scores, findings, the report and the ** page's latest results. diff --git a/src/test/java/com/kingsrook/qbits/webperf/WebPerfTablesGuardrailTest.java b/src/test/java/com/kingsrook/qbits/webperf/WebPerfTablesGuardrailTest.java new file mode 100644 index 0000000..885676c --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/webperf/WebPerfTablesGuardrailTest.java @@ -0,0 +1,185 @@ +/* + * Copyright 2026 Kingsrook, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.kingsrook.qbits.webperf; + + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Stream; +import com.kingsrook.qbits.webperf.metadata.WebPerfHelp; +import com.kingsrook.qqq.backend.core.context.QContext; +import com.kingsrook.qqq.backend.core.model.metadata.QInstance; +import com.kingsrook.qqq.backend.core.model.metadata.help.QHelpContent; +import com.kingsrook.qqq.backend.core.model.metadata.tables.QTableMetaData; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import static org.assertj.core.api.Assertions.assertThat; + + +/******************************************************************************* + ** Guardrail (owner requirement 2026-10-14: "all tables should be populated and + ** the qbit should be fully using them"). For every web_performance_* table: + ** - runtime code (not just its metadata or the home widgets) reads or writes + ** it, so no table exists only on paper; + ** - it is either seeded by the shipped changelog or listed in STARTS_EMPTY + ** with why it starts empty; + ** - it has bundle help that says what it is for and how its rows get created. + ** A new table fails this test until it is wired, seeded or explained. + *******************************************************************************/ +class WebPerfTablesGuardrailTest +{ + private static final Path MAIN = Path.of("src", "main", "java", "com", "kingsrook", "qbits", "webperf"); + + /******************************************************************************* + ** Tables the shipped changelog seeds (WebPerfDefaultsChangelogTest proves it). + *******************************************************************************/ + static final Map SEEDED = Map.of( + "budget", "changeset 10: Google's good thresholds and the Lighthouse category bands"); + + /******************************************************************************* + ** Tables that legitimately start empty, and where their rows come from. + *******************************************************************************/ + static final Map STARTS_EMPTY = Map.of( + "target", "synced from the sitemap (Sync pages from sitemap) or added by hand; the page list is host content", + "lab_run", "runtime: every queued or finished Lighthouse test", + "lab_finding", "runtime: findings of each successful lab run", + "lab_report", "runtime: the full response of each successful lab run", + "field_snapshot", "runtime: Google field data checks", + "regression", "runtime: a budget is broken or a score drifts (empty is healthy)", + "change_event", "runtime: a deploy or content change is detected (needs a probe or source)", + "comparison", "runtime: Compare before and after on a change event"); + + + + /******************************************************************************* + ** + *******************************************************************************/ + @AfterEach + void tearDown() + { + QContext.clear(); + } + + + + /******************************************************************************* + ** Every table is used by runtime code. + *******************************************************************************/ + @Test + void everyTableIsUsedAtRuntime() throws Exception + { + List runtimeSource; + try(Stream files = Files.walk(MAIN)) + { + runtimeSource = files.filter(p -> p.toString().endsWith(".java")) + .filter(p -> !p.startsWith(MAIN.resolve("metadata")) && !p.startsWith(MAIN.resolve("widgets"))) + .map(WebPerfTablesGuardrailTest::read).toList(); + } + for(String base : WebPerfSchema.TABLES) + { + Pattern use = Pattern.compile("\\{" + base + "\\}|WebPerfSchema\\." + base.toUpperCase(Locale.ROOT) + "\\b"); + assertThat(runtimeSource.stream().anyMatch(source -> use.matcher(source).find())) + .as(base + " must be read or written by runtime code, not only declared").isTrue(); + } + } + + + + /******************************************************************************* + ** Every table is seeded or explained. + *******************************************************************************/ + @Test + void everyTableIsSeededOrExplained() throws Exception + { + String changelog = Files.readString(Path.of("src", "main", "resources", WebPerfSchema.CHANGELOG), StandardCharsets.UTF_8); + for(String base : WebPerfSchema.TABLES) + { + boolean seeded = SEEDED.containsKey(base); + assertThat(seeded || STARTS_EMPTY.containsKey(base)).as(base + " must be seeded or listed in STARTS_EMPTY").isTrue(); + assertThat(seeded && STARTS_EMPTY.containsKey(base)).as(base + " cannot be both").isFalse(); + assertThat(changelog.contains("INSERT INTO ${web_performance.tablePrefix}" + base + " ")).as(base + " seeded iff the changelog inserts it").isEqualTo(seeded); + } + assertThat(WebPerfSchema.TABLES).containsAll(SEEDED.keySet()).containsAll(STARTS_EMPTY.keySet()); + } + + + + /******************************************************************************* + ** Every table and the app have bundle help that says how rows get created. + *******************************************************************************/ + @Test + void everyTableHasEmptyStateHelp() throws Exception + { + QInstance qInstance = WebPerfTestSupport.newInstance(); + new WebPerfQBitProducer().withQBitConfig(WebPerfTestSupport.config()).produce(qInstance); + WebPerfQBitConfig config = WebPerfTestSupport.config(); + + assertThat(WebPerfHelp.entries()).allSatisfy((key, entry) -> assertThat(entry.path("summary").asText().length()).as(key).isLessThanOrEqualTo(120)); + for(String base : WebPerfSchema.TABLES) + { + QTableMetaData table = qInstance.getTable(config.applyPrefix(base)); + List help = table.getHelpContent() == null ? null : table.getHelpContent().get(WebPerfHelp.SLOT); + assertThat(help).as(base + " table help").isNotEmpty(); + assertThat(help.get(0).getContent()).as(base + " help says how rows get created").contains("How rows get created"); + } + assertThat(qInstance.getApp(config.getAppName()).getHelpContent()).containsKey(WebPerfHelp.SLOT); + assertThat(WebPerfHelp.entries()).hasSize(WebPerfSchema.TABLES.size() + 1); + } + + + + /******************************************************************************* + ** Help a host already wrote is kept; the bundle only fills empty slots. + *******************************************************************************/ + @Test + void hostHelpIsNotOverridden() throws Exception + { + QInstance qInstance = WebPerfTestSupport.newInstance(); + WebPerfQBitConfig config = WebPerfTestSupport.config(); + new WebPerfQBitProducer().withQBitConfig(config).produce(qInstance); + QTableMetaData table = qInstance.getTable(config.applyPrefix(WebPerfSchema.BUDGET)); + table.getHelpContent().remove(WebPerfHelp.SLOT); + table.withHelpContent(WebPerfHelp.SLOT, new QHelpContent("host words")); + + WebPerfHelp.apply(qInstance, config); + + assertThat(table.getHelpContent().get(WebPerfHelp.SLOT).get(0).getContent()).isEqualTo("host words"); + } + + + + /******************************************************************************* + ** + *******************************************************************************/ + private static String read(Path path) + { + try + { + return (Files.readString(path, StandardCharsets.UTF_8)); + } + catch(Exception e) + { + throw (new IllegalStateException(e)); + } + } + +}