From 5b7c9474807c6d696eda1a940fd30215a5a5e5d4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 26 Oct 2026 10:33:49 -0500 Subject: [PATCH 1/6] test: add independent-node presence workload Co-Authored-By: GPT-6 (Codex) --- docs/performance/MULTI-NODE.md | 45 +++++ docs/performance/presence_nodes.py | 162 ++++++++++++++++++ .../devserver/CapacityPresenceDevServer.java | 134 +++++++++++++++ .../devserver/CollaborationDevServer.java | 28 ++- 4 files changed, 367 insertions(+), 2 deletions(-) create mode 100644 docs/performance/MULTI-NODE.md create mode 100644 docs/performance/presence_nodes.py create mode 100644 src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java diff --git a/docs/performance/MULTI-NODE.md b/docs/performance/MULTI-NODE.md new file mode 100644 index 0000000..3c067e6 --- /dev/null +++ b/docs/performance/MULTI-NODE.md @@ -0,0 +1,45 @@ +# Independent-node presence workload + +This fixture targets QQQ #1302/#1291. It runs 2–8 independent JVM HTTP servers +against an owned loopback Redis container. Every node serves the same configured +mock identities across 2–8 tenant scopes. The scope comes from the authenticated +fixture identity, never a supplied tenant parameter. Records are independently +seeded in memory; this does not qualify shared database or production IAM behavior. + +Build with Java 21: + +```powershell +./mvnw.cmd test-compile dependency:build-classpath '-Dmdep.includeScope=test' '-Dmdep.outputFile=target/capacity-classpath.txt' +python docs/performance/presence_nodes.py --java /path/to/java --nodes 3 --tenants 2 --users 6 --tabs 4 --rounds 100 +``` + +Four tabs per user means two duplicate tabs on each of two records. Every round +renews each issued lease on a rotating node, with sequential per-tab sequences. +All nodes must then return exactly the expected tenant's distinct people and +exactly two editors on both records. Any mismatch or request failure fails the +run. Requests use bounded worker concurrency and ten-second timeouts. Each node +has eight pooled Redis connections, a 500 ms pool wait and a 384 MiB heap ceiling. + +Results and per-node logs are retained under `target/capacity-nodes-`. +The result records parameters, node PIDs, elapsed time, sample counts and +nearest-rank HTTP p50/p95/p99/max. Renewal throughput includes snapshot checks +and is not maximum server throughput. All owned JVMs and the Redis container are +removed in `finally`; the receipt reports cleanup separately from test success. + +Initial smoke run: three JVMs, two tenants, six users each, four tabs per user, +three rounds: 144 renewals and 36 snapshots passed with tenant isolation, +duplicate-tab deduplication and two editors retained. Cleanup passed. This short +run validates the harness, not sustained capacity or operational thresholds. + +A second run with 12 users per tenant and 100 rounds passed 9,600 renewals and +1,200 snapshots in 23.110 seconds. Renewal p50/p95/p99 were 7.246/16.454/21.853 ms; +snapshot p50/p95/p99 were 12.908/15.776/18.239 ms. Every round checked exact +tenant membership, deduplication and two editors on both records at every node. +Cleanup passed. These are measurements from one local Windows/Docker run; +they do not establish deployment limits or browser propagation latency. + +Remaining qualification includes larger sustained workloads, CPU/memory and pool +utilization sampling, browser/SSE propagation percentiles, saturation, churn, +missed messages and process crashes. Loopback HTTP latency must not be reported +as browser propagation or a production SLA. No deployed capacity threshold is +established by this fixture. diff --git a/docs/performance/presence_nodes.py b/docs/performance/presence_nodes.py new file mode 100644 index 0000000..56bc311 --- /dev/null +++ b/docs/performance/presence_nodes.py @@ -0,0 +1,162 @@ +"""Bounded HTTP presence workload across independent, disposable JVMs. + +This measures loopback HTTP service latency, not browser propagation or a +production capacity limit. Run test-compile and dependency:build-classpath with +test scope and output target/capacity-classpath.txt before invoking this script. +""" + +import argparse +from concurrent.futures import ThreadPoolExecutor +import json +import math +import os +from pathlib import Path +import socket +import subprocess +import time +import urllib.request +import uuid + + +def port(): + """Choose a loopback port; child startup detects a competing bind.""" + with socket.socket() as listener: + listener.bind(("127.0.0.1", 0)) + return listener.getsockname()[1] + + +def request(node, user, path, body=None, tab=0): + """Use real HTTP and the explicit mock identity; no tenant parameter.""" + separator = "&" if "?" in path else "?" + req = urllib.request.Request( + f"http://127.0.0.1:{node}/qqq/v1/qbit/collaboration{path}{separator}user={user}", + data=None if body is None else json.dumps(body).encode(), + headers={"Content-Type": "application/json", + "Cookie": f"sessionUUID={user}-capacity-{tab}"}) + with urllib.request.urlopen(req, timeout=10) as response: + if response.status != 200: + raise AssertionError(f"Unexpected status {response.status}") + return json.load(response) + + +def distribution(values): + """Nearest-rank milliseconds, with no discarded slow or failed samples.""" + ordered = sorted(values) + return {"samples": len(ordered), **{ + name: ordered[max(0, math.ceil(len(ordered) * fraction) - 1)] + for name, fraction in (("p50_ms", .5), ("p95_ms", .95), ("p99_ms", .99), ("max_ms", 1))}} + + +def main(): + """Own child JVMs/container and retain results even when a check fails.""" + parser = argparse.ArgumentParser() + parser.add_argument("--java", required=True) + parser.add_argument("--nodes", type=int, default=3) + parser.add_argument("--tenants", type=int, default=2) + parser.add_argument("--users", type=int, default=6) + parser.add_argument("--tabs", type=int, default=4) + parser.add_argument("--rounds", type=int, default=10) + parser.add_argument("--workers", type=int, default=8) + args = parser.parse_args() + for name, lower, upper in (("nodes", 2, 8), ("tenants", 2, 8), ("users", 2, 100), + ("tabs", 2, 8), ("rounds", 1, 1000), ("workers", 1, 64)): + if not lower <= getattr(args, name) <= upper: + parser.error(f"{name} must be {lower}..{upper}") + repo = Path(__file__).resolve().parents[2] + output = repo / "target" / ("capacity-nodes-" + uuid.uuid4().hex[:10]) + output.mkdir(parents=True) + classpath = os.pathsep.join([str(repo / "target/test-classes"), str(repo / "target/classes"), + (repo / "target/capacity-classpath.txt").read_text().strip()]) + processes, logs = [], [] + container = "qqq-capacity-" + uuid.uuid4().hex[:12] + result = {"status": "FAIL", "parameters": vars(args), "scope": "loopback HTTP, mock identities, memory records"} + try: + subprocess.run(["docker", "run", "-d", "--name", container, "-p", "127.0.0.1::6379", + "redis:7.4-alpine"], check=True, capture_output=True) + redis_port = int(subprocess.check_output(["docker", "port", container, "6379"], text=True).strip().rsplit(":", 1)[1]) + nodes = [port() for _ in range(args.nodes)] + if len(set(nodes)) != len(nodes): + raise RuntimeError("Port allocation collision; no nodes started") + for node in nodes: + argfile = output / f"node-{node}.args" + argfile.write_text(f'-Xmx384m\n-Dcollab.capacity.tenants={args.tenants}\n-Dcollab.capacity.users={args.users}\n' + '-cp\n"' + classpath.replace("\\", "/") + '"\n' + 'com.kingsrook.qbits.collaboration.devserver.CapacityPresenceDevServer\n' + f'{node}\n{redis_port}\n', encoding="utf-8") + log = (output / f"node-{node}.log").open("w") + logs.append(log) + processes.append(subprocess.Popen([args.java, "@" + str(argfile)], cwd=repo, + stdout=log, stderr=subprocess.STDOUT, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))) + deadline = time.monotonic() + 120 + for node, process in zip(nodes, processes): + while f"CAPACITY_PRESENCE_READY {node}" not in (output / f"node-{node}.log").read_text(errors="replace"): + if process.poll() is not None or time.monotonic() > deadline: + raise RuntimeError(f"Node {node} failed startup; inspect {output}") + time.sleep(.2) + actors = [] + for tenant in range(args.tenants): + for user in range(args.users): + for tab in range(args.tabs): + identity = f"load-t{tenant}-u{user}" + record = str(1 + tab % 2) + node = nodes[len(actors) % len(nodes)] + lease = request(node, identity, "/presence/leases", {"table": "person", "pk": record}, tab)["connectionId"] + actors.append((identity, record, tab, lease, user)) + renewals, snapshots = [], [] + + def renew(item, sequence): + """Keep each tab sequential while moving its writes between nodes.""" + index, (identity, record, tab, lease, user) = item + before = time.perf_counter() + request(nodes[(index + sequence) % len(nodes)], identity, "/presence/leases/renew", + {"connectionId": lease, "table": "person", "pk": record, + "mode": "EDITING" if user < 2 else "VIEWING", "sequence": sequence}, tab) + return (time.perf_counter() - before) * 1000 + + started = time.perf_counter() + with ThreadPoolExecutor(max_workers=args.workers) as workers: + for sequence in range(1, args.rounds + 1): + renewals.extend(workers.map(lambda item: renew(item, sequence), enumerate(actors))) + for tenant in range(args.tenants): + expected = {f"load-t{tenant}-u{user}" for user in range(args.users)} + for node in nodes: + for record in ("1", "2"): + before = time.perf_counter() + viewers = request(node, f"load-t{tenant}-u0", f"/presence?table=person&pk={record}")["viewers"] + snapshots.append((time.perf_counter() - before) * 1000) + actual = {viewer["userId"] for viewer in viewers} + if actual != expected or len(viewers) != len(expected): + raise AssertionError(f"Tenant isolation/deduplication mismatch: {actual} != {expected}") + if sum(viewer["mode"] == "EDITING" for viewer in viewers) != 2: + raise AssertionError("Expected exactly two editors") + elapsed = time.perf_counter() - started + result.update(status="PASS", seconds=elapsed, renewals=distribution(renewals), + snapshots=distribution(snapshots), renewal_throughput=len(renewals) / elapsed, + node_pids=[process.pid for process in processes]) + except Exception as error: + result["error"] = str(error) + raise + finally: + for process in processes: + if process.poll() is None: + process.terminate() + for process in processes: + try: + process.wait(timeout=10) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=10) + for log in logs: + log.close() + cleanup = subprocess.run(["docker", "rm", "-f", container], capture_output=True) + result["cleanup"] = cleanup.returncode == 0 and all(process.poll() is not None for process in processes) + (output / "result.json").write_text(json.dumps(result, indent=2)) + print(json.dumps(result, indent=2)) + print(output) + if result["status"] == "PASS" and not result["cleanup"]: + raise RuntimeError("Workload passed but owned resource cleanup failed") + + +if __name__ == "__main__": + main() diff --git a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java new file mode 100644 index 0000000..7aaf535 --- /dev/null +++ b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java @@ -0,0 +1,134 @@ +/* + * QQQ - Low-code Application Framework for Engineers. + * Copyright (C) 2021-2025. Kingsrook, LLC + * 651 N Broad St Ste 205 # 6917 | Middletown DE 19709 | United States + * contact@kingsrook.com + * https://github.com/Kingsrook/ + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.kingsrook.qbits.collaboration.devserver; + +import com.kingsrook.qbits.collaboration.CollaborationQBitConfig; +import com.kingsrook.qbits.collaboration.presence.PresenceRegistry; +import com.kingsrook.qbits.collaboration.presence.PresenceScopeProvider; +import com.kingsrook.qbits.collaboration.presence.RedisPresenceConfig; +import com.kingsrook.qbits.redis.RedisConnectionConfig; +import com.kingsrook.qbits.redis.RedisQBitConfig; +import com.kingsrook.qbits.redis.RedisQBitProducer; +import com.kingsrook.qbits.redis.RedisRealtimeRuntimeService; +import com.kingsrook.qbits.redis.RedisRuntimeService; +import com.kingsrook.qqq.backend.core.model.metadata.code.QCodeReference; +import com.kingsrook.qqq.backend.core.realtime.RealtimeBus; + +/******************************************************************************* + ** Loopback-only browser acceptance node. Run separate JVMs against one disposable + ** Redis endpoint. Record data is independently seeded, not a shared database. + *******************************************************************************/ +public class CapacityPresenceDevServer +{ + /*************************************************************************** + ** Arguments are HTTP port and disposable local Redis port. Never deploy this + ** fixture: it uses the demo's mock authentication and plaintext loopback Redis. + ***************************************************************************/ + public static void main(String[] args) throws Exception + { + if(args.length != 2) + { + throw (new IllegalArgumentException("Expected HTTP port and local Redis port")); + } + int port = Integer.parseInt(args[0]); + int redisPort = Integer.parseInt(args[1]); + var instance = CollaborationDevServer.buildInstance(port); + CollaborationDevServer.seed(instance); + instance.withInstanceDefaultAuthentication(new com.kingsrook.qqq.backend.core.model.metadata.authentication.QAuthenticationMetaData() + .withName("mock").withType(com.kingsrook.qqq.backend.core.model.metadata.QAuthenticationType.MOCK) + .withCustomizer(new QCodeReference(AcceptanceSessions.class))); + PresenceRegistry.close(instance); + RealtimeBus.close(instance); + new RedisQBitProducer().withQBitConfig(new RedisQBitConfig().withConnection("presence", + new RedisConnectionConfig().withHost("127.0.0.1").withPort(redisPort).withTls(false).withMaximumConnections(8).withMaximumWaitMillis(500)) + .withRealtime("presence", "acceptance", "collaboration")).produce(instance); + instance.getQBits().values().stream().map(bit -> bit.getConfig()) + .filter(CollaborationQBitConfig.class::isInstance).map(CollaborationQBitConfig.class::cast) + .findFirst().orElseThrow().withRedisPresence(new RedisPresenceConfig().withConnectionName("presence") + .withDeployment("acceptance").withApplication("collaboration") + .withScopeProvider(new QCodeReference(FixtureTenant.class))); + var redis = new RedisRuntimeService(); + var transport = new RedisRealtimeRuntimeService(); + new com.kingsrook.qqq.backend.core.health.HealthMetaDataProvider().defineAll(instance, "memory", null); + var health = new com.kingsrook.qqq.backend.core.health.HealthRunner(instance, "acceptance-" + port, java.time.Clock.systemUTC()); + var server = CollaborationDevServer.server(instance, port, false); + try + { + redis.start(instance); + transport.start(instance); + health.start(); + server.start(); + System.out.println("CAPACITY_PRESENCE_READY " + port); + Thread.currentThread().join(); + } + finally + { + server.stop(); + health.stop(); + transport.stop(); + PresenceRegistry.close(instance); + RealtimeBus.close(instance); + redis.stop(); + redis.afterApplicationStop(); + } + } + + /*************************************************************************** + ** Tenant comes from the configured mock identity, never a request tenant parameter. + ***************************************************************************/ + public static class FixtureTenant implements PresenceScopeProvider + { + /************************************************************************ + ** Identical record IDs are isolated by the authenticated fixture identity. + ************************************************************************/ + @Override + public String tenantScope() + { + String identity = com.kingsrook.qqq.backend.core.context.QContext.getQSession().getUser().getIdReference(); + if(!identity.matches("load-t[0-7]-u[0-9]{1,2}")) + { + throw (new IllegalStateException("Capacity presence requires a configured load identity")); + } + return (identity.substring(0, identity.indexOf("-u"))); + } + } + + /*************************************************************************** + ** Grant Redis dashboard access only to the fixture's existing administrator. + ***************************************************************************/ + public static class AcceptanceSessions extends CollaborationDevServer.DemoSessions + { + /************************************************************************ + ** Keep normal demo identity/grants and add the service-view permission. + ************************************************************************/ + @Override + public void customizeSession(com.kingsrook.qqq.backend.core.model.metadata.QInstance instance, + com.kingsrook.qqq.backend.core.model.session.QSession target, java.util.Map context) + { + super.customizeSession(instance, target, context); + if("admin".equals(target.getUser().getIdReference())) + { + target.withPermissions("redisView.hasAccess", "health.read"); + } + } + } +} diff --git a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CollaborationDevServer.java b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CollaborationDevServer.java index 752c826..177d48d 100644 --- a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CollaborationDevServer.java +++ b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CollaborationDevServer.java @@ -123,11 +123,35 @@ import io.javalin.config.JavalinConfig; public class CollaborationDevServer extends AbstractQQQApplication { private final QInstance instance; - static final List USERS = List.of("alice", "bob", "admin"); + static final List USERS = fixtureUsers(); static final String USER_COOKIE = "collab_dev_user"; private static final ThreadLocal REQUEST_USER = new ThreadLocal<>(); + /******************************************************************************* + ** Opt-in bounded identities for the disposable capacity fixture only. + *******************************************************************************/ + private static List fixtureUsers() + { + int tenants = Integer.getInteger("collab.capacity.tenants", 0); + int users = Integer.getInteger("collab.capacity.users", 0); + if(tenants < 0 || tenants > 8 || users < 0 || users > 100) + { + throw (new IllegalArgumentException("Capacity fixture permits at most 8 tenants and 100 users each")); + } + var identities = new java.util.ArrayList<>(List.of("alice", "bob", "admin")); + for(int tenant = 0; tenant < tenants; tenant++) + { + for(int user = 0; user < users; user++) + { + identities.add("load-t" + tenant + "-u" + user); + } + } + return (List.copyOf(identities)); + } + + + /******************************************************************************* ** Keep the already seeded instance when Javalin initializes the application. @@ -332,7 +356,7 @@ public class CollaborationDevServer extends AbstractQQQApplication { case "alice" -> "Alice Morgan"; case "bob" -> "Bob Chen"; - default -> "Demo Administrator"; + default -> user.equals("admin") ? "Demo Administrator" : user; }; QSession session = new QSession().withUser(new QUser().withIdReference(user).withFullName(name)) .withPermissions("person.read", "person.comment") -- 2.54.0 From ea06d6fb4e671d198f883d140a190170ea20c067 Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 26 Oct 2026 11:07:11 -0500 Subject: [PATCH 2/6] test: measure presence resources and node loss Co-Authored-By: GPT-6 (Codex) --- docs/performance/MULTI-NODE.md | 83 ++++++++++++- docs/performance/presence_nodes.py | 113 ++++++++++++++++-- .../results/linux-1000-rounds.json | 101 ++++++++++++++++ .../performance/results/linux-node-crash.json | 96 +++++++++++++++ docs/performance/results/windows-timeout.json | 61 ++++++++++ .../devserver/CapacityPresenceDevServer.java | 19 ++- .../devserver/CollaborationDevServer.java | 2 +- 7 files changed, 464 insertions(+), 11 deletions(-) create mode 100644 docs/performance/results/linux-1000-rounds.json create mode 100644 docs/performance/results/linux-node-crash.json create mode 100644 docs/performance/results/windows-timeout.json diff --git a/docs/performance/MULTI-NODE.md b/docs/performance/MULTI-NODE.md index 3c067e6..653f556 100644 --- a/docs/performance/MULTI-NODE.md +++ b/docs/performance/MULTI-NODE.md @@ -26,6 +26,16 @@ nearest-rank HTTP p50/p95/p99/max. Renewal throughput includes snapshot checks and is not maximum server throughput. All owned JVMs and the Redis container are removed in `finally`; the receipt reports cleanup separately from test success. +Each fixture JVM emits resource samples every 100 ms using Java management APIs +and the Redis QBit's existing pool metrics. The receipt summarizes sampled heap +used/committed bytes, active/idle/waiting pool counts and the configured maximum. +CPU core equivalents are process CPU time divided by observed wall time; they +can exceed one for a multithreaded JVM. Observation begins after server startup +and includes lease setup and workload execution. Unsupported CPU counters are +reported as null. Heap is not total resident memory. Sampled peaks may miss +short bursts and must not be represented as exact maxima. Raw samples remain in +node logs; missing node samples or a sampled pool-bound violation fail the run. + Initial smoke run: three JVMs, two tenants, six users each, four tabs per user, three rounds: 144 renewals and 36 snapshots passed with tenant isolation, duplicate-tab deduplication and two editors retained. Cleanup passed. This short @@ -38,8 +48,77 @@ tenant membership, deduplication and two editors on both records at every node. Cleanup passed. These are measurements from one local Windows/Docker run; they do not establish deployment limits or browser propagation latency. -Remaining qualification includes larger sustained workloads, CPU/memory and pool -utilization sampling, browser/SSE propagation percentiles, saturation, churn, +## Sustained-run limitation under investigation + +The 1,000-round workload has intermittently failed with HTTP 503 on lease +renewal. Four runs failed; one completed 96,000 renewals and 12,000 snapshots. +The passing run does not resolve the failures. Temporary local diagnostics on +the third failure identified `JedisConnectionException` caused by +`SocketTimeoutException` while reading the `EVALSHA` response in +`RedisPresenceStore.mutate`. The configured socket timeout remained two seconds. +The temporary production-file diagnostics were removed after identifying this +path. No timeout increase, hidden request retry or production fix is included. + +Resource samples did not show pool waiters, but cannot rule out short bursts. +Redis slow-command timings and latency events are now captured before fixture +cleanup, alongside Redis CPU, memory and command counters, to distinguish server +execution cost from transport or scheduling delays. Slow-command summaries omit +command arguments. These diagnostics belong only to the disposable fixture. +Until the cause is established and qualified, this workload establishes no +supported sustained throughput or production alert thresholds. + +The timing-instrumented Windows-JVM run failed at round 480, with no commands +at or above the configured one-millisecond Redis slow-log threshold and no +latency-monitor events. Redis reported about 93 microseconds per `EVALSHA` on +average. This weakens a slow-Lua explanation but does not identify the delayed +component or exclude time spent outside command execution. + +## Linux comparison + +`--node-image eclipse-temurin:21-jre` optionally runs each JVM in its own Linux +container on an owned Docker network shared with Redis. The HTTP driver remains +on the host. Only loopback HTTP ports are published. Compiled fixture classes +and copied dependency JARs are mounted read-only; Redis remains disposable. +The existing loopback defaults remain unchanged when this option is absent. +Container mode records Docker client PIDs separately from container identities. +It removes the Windows-JVM-to-published-Redis-port path, but also changes the JVM +platform and scheduling environment; it is a diagnostic comparison, not proof +of a specific operating-system or Docker defect. Cleanup includes every owned +node container and the owned network. + +```powershell +python docs/performance/presence_nodes.py --java java --node-image eclipse-temurin:21-jre --nodes 3 --tenants 2 --users 12 --tabs 4 --rounds 1000 +``` + +The first Linux comparison completed 1,000 rounds in 250.811 seconds: 96,000 +renewals and 12,000 snapshots, with exact membership/editor assertions and +cleanup passing. Renewal p95/p99 were 15.331/17.760 ms. The sampled active pool +peak was five of eight, with no sampled waiters. This one run is separate from +the unresolved Windows failures and does not establish a production SLA. + +`--crash-round 50` abruptly kills the first node before that round, then sends +all existing leases through surviving nodes. It does not acquire replacement +leases or hide failed requests with retries. Exact tenant membership, duplicate +tabs and editor counts must remain correct on every surviving node. This tests +HTTP lease continuity after process loss, not browser/SSE reconnect, ingress +failover, database durability or a Redis outage. + +The Linux crash run passed 100 rounds (9,600 renewals and 996 snapshots) in +25.148 seconds. The first JVM exited with code 137 at round 50; the remaining +two retained all issued leases and expected membership. Cleanup passed. + +Recorded receipts retain both successful and failing evidence: + +- [Windows timeout with Redis diagnostics](results/windows-timeout.json) +- [Linux 1,000-round run](results/linux-1000-rounds.json) +- [Linux node crash](results/linux-node-crash.json) + +Only the Java executable path is normalized in these receipts. The Linux image +used for these runs was +`eclipse-temurin@sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99`. + +Remaining qualification includes larger sustained workloads, total resident +memory and Redis-server resource measurements, browser/SSE propagation percentiles, saturation, churn, missed messages and process crashes. Loopback HTTP latency must not be reported as browser propagation or a production SLA. No deployed capacity threshold is established by this fixture. diff --git a/docs/performance/presence_nodes.py b/docs/performance/presence_nodes.py index 56bc311..54d8ba1 100644 --- a/docs/performance/presence_nodes.py +++ b/docs/performance/presence_nodes.py @@ -11,9 +11,12 @@ import json import math import os from pathlib import Path +import shutil import socket import subprocess +import sys import time +import urllib.error import urllib.request import uuid @@ -33,7 +36,13 @@ def request(node, user, path, body=None, tab=0): data=None if body is None else json.dumps(body).encode(), headers={"Content-Type": "application/json", "Cookie": f"sessionUUID={user}-capacity-{tab}"}) - with urllib.request.urlopen(req, timeout=10) as response: + try: + response = urllib.request.urlopen(req, timeout=10) + except urllib.error.HTTPError as error: + with error: + detail = error.read(4096).decode(errors="replace") + raise RuntimeError(f"HTTP {error.code} node={node} user={user} path={path}: {detail}") from error + with response: if response.status != 200: raise AssertionError(f"Unexpected status {response.status}") return json.load(response) @@ -51,17 +60,21 @@ def main(): """Own child JVMs/container and retain results even when a check fails.""" parser = argparse.ArgumentParser() parser.add_argument("--java", required=True) + parser.add_argument("--node-image", help="Optional Linux Java21 image; isolate Redis traffic on an owned Docker network") parser.add_argument("--nodes", type=int, default=3) parser.add_argument("--tenants", type=int, default=2) parser.add_argument("--users", type=int, default=6) parser.add_argument("--tabs", type=int, default=4) parser.add_argument("--rounds", type=int, default=10) parser.add_argument("--workers", type=int, default=8) + parser.add_argument("--crash-round", type=int, default=0, help="Kill the first node before this round; zero disables") args = parser.parse_args() for name, lower, upper in (("nodes", 2, 8), ("tenants", 2, 8), ("users", 2, 100), ("tabs", 2, 8), ("rounds", 1, 1000), ("workers", 1, 64)): if not lower <= getattr(args, name) <= upper: parser.error(f"{name} must be {lower}..{upper}") + if args.crash_round and not 2 <= args.crash_round <= args.rounds: + parser.error("crash-round must be zero or 2..rounds") repo = Path(__file__).resolve().parents[2] output = repo / "target" / ("capacity-nodes-" + uuid.uuid4().hex[:10]) output.mkdir(parents=True) @@ -69,28 +82,58 @@ def main(): (repo / "target/capacity-classpath.txt").read_text().strip()]) processes, logs = [], [] container = "qqq-capacity-" + uuid.uuid4().hex[:12] + network = container + "-net" + node_containers = [] + network_created = False result = {"status": "FAIL", "parameters": vars(args), "scope": "loopback HTTP, mock identities, memory records"} try: + network_args = [] + if args.node_image: + result["node_image_id"] = subprocess.check_output( + ["docker", "image", "inspect", args.node_image, "--format", "{{.Id}}"], text=True).strip() + subprocess.run(["docker", "network", "create", network], check=True, capture_output=True) + network_created = True + network_args = ["--network", network] + dependencies = output / "dependencies" + dependencies.mkdir() + linux_classpath = ["/fixture/target/test-classes", "/fixture/target/classes"] + for index, dependency in enumerate((repo / "target/capacity-classpath.txt").read_text().strip().split(os.pathsep)): + destination = dependencies / f"{index}.jar" + shutil.copyfile(dependency, destination) + linux_classpath.append("/fixture/" + destination.relative_to(repo).as_posix()) + classpath = ":".join(linux_classpath) subprocess.run(["docker", "run", "-d", "--name", container, "-p", "127.0.0.1::6379", - "redis:7.4-alpine"], check=True, capture_output=True) + *network_args, + "redis:7.4-alpine", "redis-server", "--slowlog-log-slower-than", "1000", + "--slowlog-max-len", "128", "--latency-monitor-threshold", "1"], check=True, capture_output=True) redis_port = int(subprocess.check_output(["docker", "port", container, "6379"], text=True).strip().rsplit(":", 1)[1]) nodes = [port() for _ in range(args.nodes)] if len(set(nodes)) != len(nodes): raise RuntimeError("Port allocation collision; no nodes started") for node in nodes: argfile = output / f"node-{node}.args" + connection_properties = (f"-Dcollab.capacity.redisHost={container}\n-Dcollab.dev.bindAddress=0.0.0.0\n" + if args.node_image else "") argfile.write_text(f'-Xmx384m\n-Dcollab.capacity.tenants={args.tenants}\n-Dcollab.capacity.users={args.users}\n' + + connection_properties + '-cp\n"' + classpath.replace("\\", "/") + '"\n' 'com.kingsrook.qbits.collaboration.devserver.CapacityPresenceDevServer\n' - f'{node}\n{redis_port}\n', encoding="utf-8") + + (f'8080\n6379\n' if args.node_image else f'{node}\n{redis_port}\n'), encoding="utf-8") log = (output / f"node-{node}.log").open("w") logs.append(log) - processes.append(subprocess.Popen([args.java, "@" + str(argfile)], cwd=repo, + command = [args.java, "@" + str(argfile)] + if args.node_image: + node_container = f"{container}-node-{node}" + node_containers.append(node_container) + command = ["docker", "run", "--rm", "--name", node_container, "--network", network, + "-p", f"127.0.0.1:{node}:8080", "--mount", f"type=bind,source={repo},target=/fixture,readonly", + args.node_image, "java", "@/fixture/" + argfile.relative_to(repo).as_posix()] + processes.append(subprocess.Popen(command, cwd=repo, stdout=log, stderr=subprocess.STDOUT, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))) deadline = time.monotonic() + 120 for node, process in zip(nodes, processes): - while f"CAPACITY_PRESENCE_READY {node}" not in (output / f"node-{node}.log").read_text(errors="replace"): + while "CAPACITY_PRESENCE_READY " not in (output / f"node-{node}.log").read_text(errors="replace"): if process.poll() is not None or time.monotonic() > deadline: raise RuntimeError(f"Node {node} failed startup; inspect {output}") time.sleep(.2) @@ -117,6 +160,16 @@ def main(): started = time.perf_counter() with ThreadPoolExecutor(max_workers=args.workers) as workers: for sequence in range(1, args.rounds + 1): + result["current_round"] = sequence + if sequence == args.crash_round: + crashed = nodes.pop(0) + if args.node_image: + subprocess.run(["docker", "kill", node_containers[0]], check=True, capture_output=True) + else: + processes[0].kill() + processes[0].wait(timeout=10) + result["crash"] = {"round": sequence, "http_port": crashed, + "surviving_nodes": len(nodes), "process_exit": processes[0].returncode} renewals.extend(workers.map(lambda item: renew(item, sequence), enumerate(actors))) for tenant in range(args.tenants): expected = {f"load-t{tenant}-u{user}" for user in range(args.users)} @@ -130,14 +183,35 @@ def main(): raise AssertionError(f"Tenant isolation/deduplication mismatch: {actual} != {expected}") if sum(viewer["mode"] == "EDITING" for viewer in viewers) != 2: raise AssertionError("Expected exactly two editors") + result["completed_rounds"] = sequence elapsed = time.perf_counter() - started result.update(status="PASS", seconds=elapsed, renewals=distribution(renewals), snapshots=distribution(snapshots), renewal_throughput=len(renewals) / elapsed, - node_pids=[process.pid for process in processes]) + node_pids=[process.pid for process in processes], + pid_kind="docker-client" if args.node_image else "JVM", node_containers=node_containers) except Exception as error: result["error"] = str(error) raise finally: + failure_active = sys.exc_info()[0] is not None + try: + slow = json.loads(subprocess.check_output( + ["docker", "exec", container, "redis-cli", "--json", "SLOWLOG", "GET", "128"], text=True, timeout=10)) + result["redis_slow_commands"] = [{"unix_time": entry[1], "duration_us": entry[2], "command": entry[3][0]} + for entry in slow] + result["redis_latency_events"] = json.loads(subprocess.check_output( + ["docker", "exec", container, "redis-cli", "--json", "LATENCY", "LATEST"], text=True, timeout=10)) + for section in ("cpu", "memory", "commandstats", "stats"): + value = subprocess.check_output(["docker", "exec", container, "redis-cli", "INFO", section], text=True, timeout=10) + (output / f"redis-{section}.txt").write_text(value) + except (subprocess.SubprocessError, ValueError) as error: + result["redis_diagnostic_error"] = type(error).__name__ + node_cleanup = True + for node_container in node_containers: + removed = subprocess.run(["docker", "rm", "-f", node_container], capture_output=True) + if removed.returncode != 0: + remaining = subprocess.run(["docker", "ps", "-aq", "--filter", f"name=^{node_container}$"], capture_output=True, text=True) + node_cleanup = node_cleanup and remaining.returncode == 0 and not remaining.stdout.strip() for process in processes: if process.poll() is None: process.terminate() @@ -149,13 +223,38 @@ def main(): process.wait(timeout=10) for log in logs: log.close() + resources = [] + for node_log in sorted(output.glob("node-*.log")): + samples = [json.loads(line.split("CAPACITY_METRIC ", 1)[1]) + for line in node_log.read_text(errors="replace").splitlines() + if line.startswith("CAPACITY_METRIC ")] + if samples: + first, last = samples[0], samples[-1] + wall_ms = last["uptimeMillis"] - first["uptimeMillis"] + cpu_ns = last["cpuNanos"] - first["cpuNanos"] + resources.append({"node_log": node_log.name, "samples": len(samples), + "observed_wall_ms": wall_ms, + "cpu_seconds": cpu_ns / 1e9 if first["cpuNanos"] >= 0 else None, + "cpu_core_equivalents": cpu_ns / (wall_ms * 1e6) if wall_ms > 0 and first["cpuNanos"] >= 0 else None, + **{key + "_sampled_max": max(sample[key] for sample in samples) + for key in ("heapUsedBytes", "heapCommittedBytes", "poolActive", "poolIdle", "poolWaiting", "poolMaximum")}}) + if any(sample["poolActive"] > sample["poolMaximum"] for sample in samples): + result.update(status="FAIL", error="Sampled pool active count exceeded configured maximum") + result["resources"] = resources + if result["status"] == "PASS" and len(resources) != args.nodes: + result.update(status="FAIL", error="Missing resource samples for a node") cleanup = subprocess.run(["docker", "rm", "-f", container], capture_output=True) - result["cleanup"] = cleanup.returncode == 0 and all(process.poll() is not None for process in processes) + network_cleanup = True + if network_created: + network_cleanup = subprocess.run(["docker", "network", "rm", network], capture_output=True).returncode == 0 + result["cleanup"] = cleanup.returncode == 0 and node_cleanup and network_cleanup and all(process.poll() is not None for process in processes) (output / "result.json").write_text(json.dumps(result, indent=2)) print(json.dumps(result, indent=2)) print(output) if result["status"] == "PASS" and not result["cleanup"]: raise RuntimeError("Workload passed but owned resource cleanup failed") + if result["status"] == "FAIL" and not failure_active: + raise RuntimeError(result.get("error", "Capacity resource validation failed")) if __name__ == "__main__": diff --git a/docs/performance/results/linux-1000-rounds.json b/docs/performance/results/linux-1000-rounds.json new file mode 100644 index 0000000..995a222 --- /dev/null +++ b/docs/performance/results/linux-1000-rounds.json @@ -0,0 +1,101 @@ +{ + "status": "PASS", + "parameters": { + "java": "java (Java 21)", + "node_image": "eclipse-temurin:21-jre", + "nodes": 3, + "tenants": 2, + "users": 12, + "tabs": 4, + "rounds": 1000, + "workers": 8 + }, + "scope": "loopback HTTP, mock identities, memory records", + "current_round": 1000, + "completed_rounds": 1000, + "seconds": 250.81072249996942, + "renewals": { + "samples": 96000, + "p50_ms": 6.353199947625399, + "p95_ms": 15.330899972468615, + "p99_ms": 17.760399961844087, + "max_ms": 326.8311999272555 + }, + "snapshots": { + "samples": 12000, + "p50_ms": 13.105699908919632, + "p95_ms": 14.261199976317585, + "p99_ms": 16.519399941898882, + "max_ms": 915.3468000004068 + }, + "renewal_throughput": 382.7587554595546, + "node_pids": [ + 64000, + 40240, + 115400 + ], + "pid_kind": "docker-client", + "node_containers": [ + "qqq-capacity-0a8b7956499d-node-64951", + "qqq-capacity-0a8b7956499d-node-64952", + "qqq-capacity-0a8b7956499d-node-64953" + ], + "redis_slow_commands": [ + { + "unix_time": 1793030517, + "duration_us": 1117, + "command": "EVALSHA" + } + ], + "redis_latency_events": [ + [ + "command", + 1793030517, + 1, + 1 + ] + ], + "resources": [ + { + "node_log": "node-64951.log", + "samples": 2580, + "observed_wall_ms": 257899, + "cpu_seconds": 165.55, + "cpu_core_equivalents": 0.6419179601316795, + "heapUsedBytes_sampled_max": 166667368, + "heapCommittedBytes_sampled_max": 188743680, + "poolActive_sampled_max": 4, + "poolIdle_sampled_max": 5, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-64952.log", + "samples": 2569, + "observed_wall_ms": 256799, + "cpu_seconds": 165.85, + "cpu_core_equivalents": 0.6458358482704372, + "heapUsedBytes_sampled_max": 169216944, + "heapCommittedBytes_sampled_max": 192937984, + "poolActive_sampled_max": 5, + "poolIdle_sampled_max": 4, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-64953.log", + "samples": 2608, + "observed_wall_ms": 260700, + "cpu_seconds": 164.72, + "cpu_core_equivalents": 0.631837360951285, + "heapUsedBytes_sampled_max": 166319008, + "heapCommittedBytes_sampled_max": 188743680, + "poolActive_sampled_max": 4, + "poolIdle_sampled_max": 5, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + } + ], + "cleanup": true, + "receipt_note": "Java executable path normalized; measurements unchanged. Disposable fixture identities only." +} diff --git a/docs/performance/results/linux-node-crash.json b/docs/performance/results/linux-node-crash.json new file mode 100644 index 0000000..bb87b25 --- /dev/null +++ b/docs/performance/results/linux-node-crash.json @@ -0,0 +1,96 @@ +{ + "status": "PASS", + "parameters": { + "java": "java (Java 21)", + "node_image": "eclipse-temurin@sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99", + "nodes": 3, + "tenants": 2, + "users": 12, + "tabs": 4, + "rounds": 100, + "workers": 8, + "crash_round": 50 + }, + "scope": "loopback HTTP, mock identities, memory records", + "node_image_id": "sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99", + "current_round": 100, + "completed_rounds": 100, + "crash": { + "round": 50, + "http_port": 54039, + "surviving_nodes": 2, + "process_exit": 137 + }, + "seconds": 25.147547999978997, + "renewals": { + "samples": 9600, + "p50_ms": 7.125200005248189, + "p95_ms": 17.407900071702898, + "p99_ms": 27.217599912546575, + "max_ms": 74.53640003222972 + }, + "snapshots": { + "samples": 996, + "p50_ms": 13.538200058974326, + "p95_ms": 15.59239998459816, + "p99_ms": 20.58999997097999, + "max_ms": 315.0679999962449 + }, + "renewal_throughput": 381.7469599822622, + "node_pids": [ + 85832, + 90280, + 67764 + ], + "pid_kind": "docker-client", + "node_containers": [ + "qqq-capacity-e599dca8e163-node-54039", + "qqq-capacity-e599dca8e163-node-54040", + "qqq-capacity-e599dca8e163-node-54041" + ], + "redis_slow_commands": [], + "redis_latency_events": [], + "resources": [ + { + "node_log": "node-54039.log", + "samples": 198, + "observed_wall_ms": 19699, + "cpu_seconds": 28.58, + "cpu_core_equivalents": 1.4508350677699375, + "heapUsedBytes_sampled_max": 163478640, + "heapCommittedBytes_sampled_max": 188743680, + "poolActive_sampled_max": 3, + "poolIdle_sampled_max": 4, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-54040.log", + "samples": 300, + "observed_wall_ms": 29900, + "cpu_seconds": 41.85, + "cpu_core_equivalents": 1.399665551839465, + "heapUsedBytes_sampled_max": 165177008, + "heapCommittedBytes_sampled_max": 188743680, + "poolActive_sampled_max": 4, + "poolIdle_sampled_max": 5, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-54041.log", + "samples": 331, + "observed_wall_ms": 33000, + "cpu_seconds": 41.52, + "cpu_core_equivalents": 1.2581818181818183, + "heapUsedBytes_sampled_max": 168546936, + "heapCommittedBytes_sampled_max": 192937984, + "poolActive_sampled_max": 4, + "poolIdle_sampled_max": 5, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + } + ], + "cleanup": true, + "receipt_note": "Java executable path normalized; measurements unchanged. Disposable fixture identities only." +} diff --git a/docs/performance/results/windows-timeout.json b/docs/performance/results/windows-timeout.json new file mode 100644 index 0000000..9551682 --- /dev/null +++ b/docs/performance/results/windows-timeout.json @@ -0,0 +1,61 @@ +{ + "status": "FAIL", + "parameters": { + "java": "java (Java 21)", + "nodes": 3, + "tenants": 2, + "users": 12, + "tabs": 4, + "rounds": 1000, + "workers": 8 + }, + "scope": "loopback HTTP, mock identities, memory records", + "current_round": 480, + "completed_rounds": 479, + "error": "HTTP 503 node=56028 user=load-t0-u5 path=/presence/leases/renew: {\"error\":\"Presence is temporarily unavailable. Please retry.\",\"code\":\"ERROR\",\"message\":\"Presence is temporarily unavailable. Please retry.\",\"traceId\":\"cea30b82-e82b-49ec-af8a-64d8b6bd5af6\"}", + "redis_slow_commands": [], + "redis_latency_events": [], + "resources": [ + { + "node_log": "node-56027.log", + "samples": 1067, + "observed_wall_ms": 106600, + "cpu_seconds": 94.328125, + "cpu_core_equivalents": 0.8848792213883677, + "heapUsedBytes_sampled_max": 160559696, + "heapCommittedBytes_sampled_max": 178257920, + "poolActive_sampled_max": 5, + "poolIdle_sampled_max": 5, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-56028.log", + "samples": 1066, + "observed_wall_ms": 106501, + "cpu_seconds": 95.8125, + "cpu_core_equivalents": 0.8996394400052582, + "heapUsedBytes_sampled_max": 160251104, + "heapCommittedBytes_sampled_max": 178257920, + "poolActive_sampled_max": 5, + "poolIdle_sampled_max": 4, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-56029.log", + "samples": 1066, + "observed_wall_ms": 106500, + "cpu_seconds": 95.25, + "cpu_core_equivalents": 0.8943661971830986, + "heapUsedBytes_sampled_max": 157995216, + "heapCommittedBytes_sampled_max": 175112192, + "poolActive_sampled_max": 5, + "poolIdle_sampled_max": 4, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + } + ], + "cleanup": true, + "receipt_note": "Java executable path normalized; measurements unchanged. Disposable fixture identities only." +} diff --git a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java index 7aaf535..19bb859 100644 --- a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java +++ b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java @@ -59,7 +59,7 @@ public class CapacityPresenceDevServer PresenceRegistry.close(instance); RealtimeBus.close(instance); new RedisQBitProducer().withQBitConfig(new RedisQBitConfig().withConnection("presence", - new RedisConnectionConfig().withHost("127.0.0.1").withPort(redisPort).withTls(false).withMaximumConnections(8).withMaximumWaitMillis(500)) + new RedisConnectionConfig().withHost(System.getProperty("collab.capacity.redisHost", "127.0.0.1")).withPort(redisPort).withTls(false).withMaximumConnections(8).withMaximumWaitMillis(500)) .withRealtime("presence", "acceptance", "collaboration")).produce(instance); instance.getQBits().values().stream().map(bit -> bit.getConfig()) .filter(CollaborationQBitConfig.class::isInstance).map(CollaborationQBitConfig.class::cast) @@ -71,17 +71,34 @@ public class CapacityPresenceDevServer new com.kingsrook.qqq.backend.core.health.HealthMetaDataProvider().defineAll(instance, "memory", null); var health = new com.kingsrook.qqq.backend.core.health.HealthRunner(instance, "acceptance-" + port, java.time.Clock.systemUTC()); var server = CollaborationDevServer.server(instance, port, false); + var sampler = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(); try { redis.start(instance); transport.start(instance); health.start(); server.start(); + sampler.scheduleAtFixedRate(() -> + { + var pool = RedisRuntimeService.connection(instance, "presence").metrics(); + var heap = java.lang.management.ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + var values = new java.util.LinkedHashMap(); + values.put("uptimeMillis", java.lang.management.ManagementFactory.getRuntimeMXBean().getUptime()); + values.put("cpuNanos", ProcessHandle.current().info().totalCpuDuration().map(java.time.Duration::toNanos).orElse(-1L)); + values.put("heapUsedBytes", heap.getUsed()); + values.put("heapCommittedBytes", heap.getCommitted()); + values.put("poolActive", pool.active()); + values.put("poolIdle", pool.idle()); + values.put("poolWaiting", pool.waiting()); + values.put("poolMaximum", pool.maximum()); + System.out.println("CAPACITY_METRIC " + com.kingsrook.qqq.backend.core.utils.JsonUtils.toJson(values)); + }, 0, 100, java.util.concurrent.TimeUnit.MILLISECONDS); System.out.println("CAPACITY_PRESENCE_READY " + port); Thread.currentThread().join(); } finally { + sampler.shutdownNow(); server.stop(); health.stop(); transport.stop(); diff --git a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CollaborationDevServer.java b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CollaborationDevServer.java index 177d48d..16bed25 100644 --- a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CollaborationDevServer.java +++ b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CollaborationDevServer.java @@ -210,7 +210,7 @@ public class CollaborationDevServer extends AbstractQQQApplication .withServeLegacyUnversionedMiddlewareAPI(false).withStrictUiApiRoutes(true) .withJavalinConfigCustomizer(config -> { - config.jetty.host = "127.0.0.1"; + config.jetty.host = System.getProperty("collab.dev.bindAddress", "127.0.0.1"); userSelection(config); MemoryStorage.routes(config); })); -- 2.54.0 From fd8d7afe769b029cb0b43f45532ea3da09a2b41c Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 26 Oct 2026 11:13:53 -0500 Subject: [PATCH 3/6] test: measure cross-node browser presence updates Co-Authored-By: GPT-6 (Codex) --- docs/performance/MULTI-NODE.md | 29 ++ docs/performance/browser_presence_latency.mjs | 78 +++++ .../performance/results/browser-focus-25.json | 285 ++++++++++++++++++ 3 files changed, 392 insertions(+) create mode 100644 docs/performance/browser_presence_latency.mjs create mode 100644 docs/performance/results/browser-focus-25.json diff --git a/docs/performance/MULTI-NODE.md b/docs/performance/MULTI-NODE.md index 653f556..c4a3733 100644 --- a/docs/performance/MULTI-NODE.md +++ b/docs/performance/MULTI-NODE.md @@ -117,6 +117,35 @@ Only the Java executable path is normalized in these receipts. The Linux image used for these runs was `eclipse-temurin@sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99`. +## Browser propagation baseline + +`browser_presence_latency.mjs` measures Chromium field-focus changes until a +different editor sees the corresponding remote marker. Supply a caller-owned +two-node loopback fixture serving the published frontend with `alice`/`admin` +mock identities and the acceptance router's `presence_node=first/second` +affinity cookies: + +```powershell +node docs/performance/browser_presence_latency.mjs /path/to/frontend http://localhost:PORT result.json 25 +``` + +The test verifies the target marker is absent before every change, the old +marker disappears afterward, both real SSE streams opened successfully, +issued-lease writes reached distinct nodes, and unsaved drafts survive. API +responses are not mocked. The fixture owns backend/router/Redis cleanup; this +script owns and closes its browser. + +The [25-transition receipt](results/browser-focus-25.json) passed against the +downloaded public RC37 JAR and published backend snapshots. Focus-to-observed-DOM +p50/p95/p99 were 31.648/33.078/36.092 ms; p99 is the maximum in this small sample. +There were no page errors or failed lease responses. Six-user presence and Redis +admin browser checks also passed in the same fixture run; fixture cleanup passed. + +This baseline includes browser automation, UI debounce, network, invalidation +and rendering. It is not a server-only transport measurement, an SSE-only causal +proof, an under-load result, or a production latency objective. Browser propagation +under sustained load, churn and failures remains unqualified. + Remaining qualification includes larger sustained workloads, total resident memory and Redis-server resource measurements, browser/SSE propagation percentiles, saturation, churn, missed messages and process crashes. Loopback HTTP latency must not be reported diff --git a/docs/performance/browser_presence_latency.mjs b/docs/performance/browser_presence_latency.mjs new file mode 100644 index 0000000..b3a6190 --- /dev/null +++ b/docs/performance/browser_presence_latency.mjs @@ -0,0 +1,78 @@ +// Real UI observations against a caller-owned two-node acceptance fixture. +import { createRequire } from 'node:module'; +import { resolve } from 'node:path'; +import { writeFile } from 'node:fs/promises'; +import { performance } from 'node:perf_hooks'; + +const [frontendRepo, base, output, countValue = '25'] = process.argv.slice(2); +const samples = Number(countValue); +if (!frontendRepo || !base || !output || !Number.isInteger(samples) || samples < 5 || samples > 100) { + throw new Error('Expected frontend repo, loopback base URL, result path and 5..100 samples'); +} +if (!['localhost', '127.0.0.1'].includes(new URL(base).hostname)) { + throw new Error('Only the disposable loopback fixture is supported'); +} +const require = createRequire(resolve(frontendRepo, 'package.json')); +const { chromium, expect } = require('@playwright/test'); +const result = { status: 'FAIL', samples: [], streams: [], requests: [], errors: [], + scope: 'Chromium focus-to-remote-DOM observation, including automation and UI debounce; no concurrent load' }; +const browser = await chromium.launch({ headless: true }); +try { + async function editor(user, node) { + const context = await browser.newContext(); + await context.addCookies([ + { name: 'collab_dev_user', value: user, url: base }, + { name: 'presence_node', value: node, url: base }, + ]); + const page = await context.newPage(); + page.setDefaultTimeout(15000); + page.on('pageerror', error => result.errors.push(error.message)); + page.on('response', response => { + const path = new URL(response.url()).pathname; + if (path.endsWith('/realtime/stream')) { + result.streams.push({ user, status: response.status(), type: response.headers()['content-type'] }); + } + if (path.includes('/presence/leases')) { + result.requests.push({ user, status: response.status(), node: response.headers()['x-acceptance-node'] }); + } + }); + await page.goto(`${base}/app/person/1/edit?createMockSession=true`); + await expect(page.locator('[data-qqq-id="firstName"]')).toBeVisible(); + return page; + } + const observer = await editor('alice', 'first'); + const sender = await editor('admin', 'second'); + await observer.locator('[data-qqq-id="firstName"]').fill('Retained observer draft'); + await sender.locator('[data-qqq-id="firstName"]').fill('Retained sender draft'); + await expect(observer.locator('[data-presence-field="firstName"]')).toContainText('Demo Administrator'); + for (let index = 0; index < samples; index++) { + const field = index % 2 === 0 ? 'lastName' : 'firstName'; + const previous = field === 'firstName' ? 'lastName' : 'firstName'; + await expect(observer.locator(`[data-presence-field="${field}"]`)).not.toContainText('Demo Administrator'); + const before = performance.now(); + await sender.locator(`[data-qqq-id="${field}"]`).focus(); + await expect(observer.locator(`[data-presence-field="${field}"]`)).toContainText('Demo Administrator'); + result.samples.push({ field, propagation_ms: performance.now() - before }); + await expect(observer.locator(`[data-presence-field="${previous}"]`)).not.toContainText('Demo Administrator'); + } + await expect(observer.locator('[data-qqq-id="firstName"]')).toHaveValue('Retained observer draft'); + await expect(sender.locator('[data-qqq-id="firstName"]')).toHaveValue('Retained sender draft'); + for (const user of ['alice', 'admin']) { + expect(result.streams.some(stream => stream.user === user && stream.status === 200 && stream.type?.includes('text/event-stream'))).toBe(true); + } + expect(result.requests.some(request => request.user === 'alice' && request.node === 'first')).toBe(true); + expect(result.requests.some(request => request.user === 'admin' && request.node === 'second')).toBe(true); + expect(result.requests.every(request => request.status === 200)).toBe(true); + expect(result.errors).toEqual([]); + const ordered = result.samples.map(sample => sample.propagation_ms).sort((a, b) => a - b); + result.percentiles_ms = Object.fromEntries([['p50', .5], ['p95', .95], ['p99', .99], ['max', 1]] + .map(([name, quantile]) => [name, ordered[Math.ceil(ordered.length * quantile) - 1]])); + result.status = 'PASS'; +} catch (error) { + result.failure = String(error); + process.exitCode = 1; +} finally { + await browser.close(); + await writeFile(output, JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); +} diff --git a/docs/performance/results/browser-focus-25.json b/docs/performance/results/browser-focus-25.json new file mode 100644 index 0000000..cf81846 --- /dev/null +++ b/docs/performance/results/browser-focus-25.json @@ -0,0 +1,285 @@ +{ + "status": "PASS", + "samples": [ + { + "field": "lastName", + "propagation_ms": 36.092099999999846 + }, + { + "field": "firstName", + "propagation_ms": 32.51870000000008 + }, + { + "field": "lastName", + "propagation_ms": 31.758600000000115 + }, + { + "field": "firstName", + "propagation_ms": 31.090299999999843 + }, + { + "field": "lastName", + "propagation_ms": 31.64980000000014 + }, + { + "field": "firstName", + "propagation_ms": 31.648199999999633 + }, + { + "field": "lastName", + "propagation_ms": 31.395500000000084 + }, + { + "field": "firstName", + "propagation_ms": 32.36750000000029 + }, + { + "field": "lastName", + "propagation_ms": 31.601799999999912 + }, + { + "field": "firstName", + "propagation_ms": 31.90599999999995 + }, + { + "field": "lastName", + "propagation_ms": 31.44150000000036 + }, + { + "field": "firstName", + "propagation_ms": 31.186699999999746 + }, + { + "field": "lastName", + "propagation_ms": 31.115800000000036 + }, + { + "field": "firstName", + "propagation_ms": 32.539400000000114 + }, + { + "field": "lastName", + "propagation_ms": 31.188000000000102 + }, + { + "field": "firstName", + "propagation_ms": 33.021200000000135 + }, + { + "field": "lastName", + "propagation_ms": 33.07759999999962 + }, + { + "field": "firstName", + "propagation_ms": 32.7176000000004 + }, + { + "field": "lastName", + "propagation_ms": 31.457300000000032 + }, + { + "field": "firstName", + "propagation_ms": 32.920900000000074 + }, + { + "field": "lastName", + "propagation_ms": 31.89130000000023 + }, + { + "field": "firstName", + "propagation_ms": 30.847200000000157 + }, + { + "field": "lastName", + "propagation_ms": 30.158300000000054 + }, + { + "field": "firstName", + "propagation_ms": 31.48239999999987 + }, + { + "field": "lastName", + "propagation_ms": 31.337600000000293 + } + ], + "streams": [ + { + "user": "alice", + "status": 200, + "type": "text/event-stream;charset=utf-8" + }, + { + "user": "admin", + "status": 200, + "type": "text/event-stream;charset=utf-8" + } + ], + "requests": [ + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + } + ], + "errors": [], + "scope": "Chromium focus-to-remote-DOM observation, including automation and UI debounce; no concurrent load", + "percentiles_ms": { + "p50": 31.648199999999633, + "p95": 33.07759999999962, + "p99": 36.092099999999846, + "max": 36.092099999999846 + }, + "frontend_jar_sha256": "c9fa2eb2381a47c8104f82f4e940d2bfa8ba67e2341298dde0de4e21ad3ff1ab", + "fixture_cleanup": true, + "backend_source": "Published core cc3981bb3/collaboration ecff347/Redis7fe5492 snapshots; two independent JVMs, mock identities and memory records." +} -- 2.54.0 From f38dc29f6fa9f8a07858c39b93b254b1542a6ef6 Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 26 Oct 2026 11:23:29 -0500 Subject: [PATCH 4/6] test: verify presence churn across surviving nodes Co-Authored-By: GPT-6 (Codex) --- docs/performance/MULTI-NODE.md | 14 +++ docs/performance/presence_nodes.py | 60 ++++++--- .../results/linux-node-crash-churn.json | 117 ++++++++++++++++++ 3 files changed, 177 insertions(+), 14 deletions(-) create mode 100644 docs/performance/results/linux-node-crash-churn.json diff --git a/docs/performance/MULTI-NODE.md b/docs/performance/MULTI-NODE.md index c4a3733..ff4e118 100644 --- a/docs/performance/MULTI-NODE.md +++ b/docs/performance/MULTI-NODE.md @@ -86,6 +86,13 @@ platform and scheduling environment; it is a diagnostic comparison, not proof of a specific operating-system or Docker defect. Cleanup includes every owned node container and the owned network. +`--churn-every 20` departs all tabs belonging to the last user in each tenant +every 20 rounds. Every departed lease must reject a later renewal with HTTP 409 +and `PRESENCE_LEASE_EXPIRED` on another node. All surviving nodes must show that +user absent on both records, then show the original exact membership after +fresh, distinct leases are acquired and activated. Only this explicit rejoin +path acquires new leases; the node-crash path preserves existing ones. + ```powershell python docs/performance/presence_nodes.py --java java --node-image eclipse-temurin:21-jre --nodes 3 --tenants 2 --users 12 --tabs 4 --rounds 1000 ``` @@ -107,6 +114,13 @@ The Linux crash run passed 100 rounds (9,600 renewals and 996 snapshots) in 25.148 seconds. The first JVM exited with code 137 at round 50; the remaining two retained all issued leases and expected membership. Cleanup passed. +A [combined churn and node-loss run](results/linux-node-crash-churn.json) passed +100 rounds with the first node killed at round 50 and churn every 20 rounds. +Five cycles rejected all 40 departed handles and checked absence/restored +membership across both tenants and records on every surviving node. Cleanup +passed. This is bounded HTTP lifecycle qualification; browser reconnect and +missed-event behavior remain separate requirements. + Recorded receipts retain both successful and failing evidence: - [Windows timeout with Redis diagnostics](results/windows-timeout.json) diff --git a/docs/performance/presence_nodes.py b/docs/performance/presence_nodes.py index 54d8ba1..21d1b49 100644 --- a/docs/performance/presence_nodes.py +++ b/docs/performance/presence_nodes.py @@ -28,22 +28,26 @@ def port(): return listener.getsockname()[1] -def request(node, user, path, body=None, tab=0): +def request(node, user, path, body=None, tab=0, method=None, expected=200): """Use real HTTP and the explicit mock identity; no tenant parameter.""" separator = "&" if "?" in path else "?" req = urllib.request.Request( f"http://127.0.0.1:{node}/qqq/v1/qbit/collaboration{path}{separator}user={user}", data=None if body is None else json.dumps(body).encode(), + method=method, headers={"Content-Type": "application/json", "Cookie": f"sessionUUID={user}-capacity-{tab}"}) try: response = urllib.request.urlopen(req, timeout=10) except urllib.error.HTTPError as error: + if error.code == expected: + with error: + return json.load(error) with error: detail = error.read(4096).decode(errors="replace") raise RuntimeError(f"HTTP {error.code} node={node} user={user} path={path}: {detail}") from error with response: - if response.status != 200: + if response.status != expected: raise AssertionError(f"Unexpected status {response.status}") return json.load(response) @@ -68,6 +72,7 @@ def main(): parser.add_argument("--rounds", type=int, default=10) parser.add_argument("--workers", type=int, default=8) parser.add_argument("--crash-round", type=int, default=0, help="Kill the first node before this round; zero disables") + parser.add_argument("--churn-every", type=int, default=0, help="Leave/rejoin the last user's tabs every N rounds; zero disables") args = parser.parse_args() for name, lower, upper in (("nodes", 2, 8), ("tenants", 2, 8), ("users", 2, 100), ("tabs", 2, 8), ("rounds", 1, 1000), ("workers", 1, 64)): @@ -75,6 +80,8 @@ def main(): parser.error(f"{name} must be {lower}..{upper}") if args.crash_round and not 2 <= args.crash_round <= args.rounds: parser.error("crash-round must be zero or 2..rounds") + if args.churn_every and not 1 <= args.churn_every <= args.rounds: + parser.error("churn-every must be zero or 1..rounds") repo = Path(__file__).resolve().parents[2] output = repo / "target" / ("capacity-nodes-" + uuid.uuid4().hex[:10]) output.mkdir(parents=True) @@ -148,6 +155,21 @@ def main(): actors.append((identity, record, tab, lease, user)) renewals, snapshots = [], [] + def check_rooms(excluded=None): + """Check every tenant/record on every surviving node, including absence.""" + for tenant in range(args.tenants): + expected_users = {f"load-t{tenant}-u{user}" for user in range(args.users) if user != excluded} + for node in nodes: + for record in ("1", "2"): + before = time.perf_counter() + viewers = request(node, f"load-t{tenant}-u0", f"/presence?table=person&pk={record}")["viewers"] + snapshots.append((time.perf_counter() - before) * 1000) + actual = {viewer["userId"] for viewer in viewers} + if actual != expected_users or len(viewers) != len(expected_users): + raise AssertionError(f"Tenant isolation/deduplication mismatch: {actual} != {expected_users}") + if sum(viewer["mode"] == "EDITING" for viewer in viewers) != sum(user != excluded for user in range(2)): + raise AssertionError("Unexpected editor count") + def renew(item, sequence): """Keep each tab sequential while moving its writes between nodes.""" index, (identity, record, tab, lease, user) = item @@ -171,18 +193,28 @@ def main(): result["crash"] = {"round": sequence, "http_port": crashed, "surviving_nodes": len(nodes), "process_exit": processes[0].returncode} renewals.extend(workers.map(lambda item: renew(item, sequence), enumerate(actors))) - for tenant in range(args.tenants): - expected = {f"load-t{tenant}-u{user}" for user in range(args.users)} - for node in nodes: - for record in ("1", "2"): - before = time.perf_counter() - viewers = request(node, f"load-t{tenant}-u0", f"/presence?table=person&pk={record}")["viewers"] - snapshots.append((time.perf_counter() - before) * 1000) - actual = {viewer["userId"] for viewer in viewers} - if actual != expected or len(viewers) != len(expected): - raise AssertionError(f"Tenant isolation/deduplication mismatch: {actual} != {expected}") - if sum(viewer["mode"] == "EDITING" for viewer in viewers) != 2: - raise AssertionError("Expected exactly two editors") + check_rooms() + if args.churn_every and sequence % args.churn_every == 0: + departing = [(index, actor) for index, actor in enumerate(actors) if actor[4] == args.users - 1] + for index, (identity, record, tab, lease, user) in departing: + request(nodes[index % len(nodes)], identity, "/presence", + {"connectionId": lease, "table": "person", "pk": record, "sequence": sequence + 1}, tab, method="DELETE") + response = request(nodes[(index + 1) % len(nodes)], identity, "/presence/leases/renew", + {"connectionId": lease, "table": "person", "pk": record, + "mode": "VIEWING", "sequence": sequence + 2}, tab, expected=409) + if response.get("code") != "PRESENCE_LEASE_EXPIRED": + raise AssertionError("Departed lease was not rejected as expired") + check_rooms(excluded=args.users - 1) + for index, (identity, record, tab, lease, user) in departing: + replacement = request(nodes[(index + 1) % len(nodes)], identity, "/presence/leases", + {"table": "person", "pk": record}, tab)["connectionId"] + if replacement == lease: + raise AssertionError("Rejoin reused departed authority") + actors[index] = (identity, record, tab, replacement, user) + renew((index, actors[index]), sequence) + check_rooms() + result["churn_cycles"] = result.get("churn_cycles", 0) + 1 + result["departed_leases_rejected"] = result.get("departed_leases_rejected", 0) + len(departing) result["completed_rounds"] = sequence elapsed = time.perf_counter() - started result.update(status="PASS", seconds=elapsed, renewals=distribution(renewals), diff --git a/docs/performance/results/linux-node-crash-churn.json b/docs/performance/results/linux-node-crash-churn.json new file mode 100644 index 0000000..9e2db3e --- /dev/null +++ b/docs/performance/results/linux-node-crash-churn.json @@ -0,0 +1,117 @@ +{ + "status": "PASS", + "parameters": { + "java": "java (Java 21)", + "node_image": "eclipse-temurin@sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99", + "nodes": 3, + "tenants": 2, + "users": 12, + "tabs": 4, + "rounds": 100, + "workers": 8, + "crash_round": 50, + "churn_every": 20 + }, + "scope": "loopback HTTP, mock identities, memory records", + "node_image_id": "sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99", + "current_round": 100, + "completed_rounds": 100, + "churn_cycles": 5, + "departed_leases_rejected": 40, + "crash": { + "round": 50, + "http_port": 52824, + "surviving_nodes": 2, + "process_exit": 137 + }, + "seconds": 28.090077199973166, + "renewals": { + "samples": 9600, + "p50_ms": 7.532799965701997, + "p95_ms": 17.53149996511638, + "p99_ms": 29.359900043345988, + "max_ms": 88.13189994543791 + }, + "snapshots": { + "samples": 1092, + "p50_ms": 13.582199928350747, + "p95_ms": 15.386099927127361, + "p99_ms": 18.502700026147068, + "max_ms": 22.144700051285326 + }, + "renewal_throughput": 341.75769371004685, + "node_pids": [ + 22844, + 66652, + 61920 + ], + "pid_kind": "docker-client", + "node_containers": [ + "qqq-capacity-bb9ba87919d2-node-52824", + "qqq-capacity-bb9ba87919d2-node-52825", + "qqq-capacity-bb9ba87919d2-node-52827" + ], + "redis_slow_commands": [ + { + "unix_time": 1793031681, + "duration_us": 1206, + "command": "EVALSHA" + }, + { + "unix_time": 1793031681, + "duration_us": 1147, + "command": "EVALSHA" + } + ], + "redis_latency_events": [ + [ + "command", + 1793031681, + 1, + 1 + ] + ], + "resources": [ + { + "node_log": "node-52824.log", + "samples": 212, + "observed_wall_ms": 21099, + "cpu_seconds": 29.48, + "cpu_core_equivalents": 1.3972226171856486, + "heapUsedBytes_sampled_max": 163641024, + "heapCommittedBytes_sampled_max": 188743680, + "poolActive_sampled_max": 3, + "poolIdle_sampled_max": 4, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-52825.log", + "samples": 354, + "observed_wall_ms": 35299, + "cpu_seconds": 43.09, + "cpu_core_equivalents": 1.22071446783195, + "heapUsedBytes_sampled_max": 168178592, + "heapCommittedBytes_sampled_max": 192937984, + "poolActive_sampled_max": 5, + "poolIdle_sampled_max": 5, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-52827.log", + "samples": 340, + "observed_wall_ms": 33900, + "cpu_seconds": 42.42, + "cpu_core_equivalents": 1.2513274336283187, + "heapUsedBytes_sampled_max": 166291160, + "heapCommittedBytes_sampled_max": 188743680, + "poolActive_sampled_max": 5, + "poolIdle_sampled_max": 5, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8 + } + ], + "cleanup": true, + "receipt_note": "Java executable path normalized; measurements unchanged. Disposable fixture identities only." +} -- 2.54.0 From 347b4ff6951e0b378a508451834142869f6bd73d Mon Sep 17 00:00:00 2001 From: James Maes Date: Mon, 26 Oct 2026 11:28:40 -0500 Subject: [PATCH 5/6] test: verify bounded Redis stall recovery Co-Authored-By: GPT-6 (Codex) --- docs/performance/MULTI-NODE.md | 16 +++ docs/performance/presence_nodes.py | 37 ++++++ .../results/linux-redis-stall.json | 117 ++++++++++++++++++ 3 files changed, 170 insertions(+) create mode 100644 docs/performance/results/linux-redis-stall.json diff --git a/docs/performance/MULTI-NODE.md b/docs/performance/MULTI-NODE.md index ff4e118..0445077 100644 --- a/docs/performance/MULTI-NODE.md +++ b/docs/performance/MULTI-NODE.md @@ -93,6 +93,22 @@ user absent on both records, then show the original exact membership after fresh, distinct leases are acquired and activated. Only this explicit rejoin path acquires new leases; the node-crash path preserves existing ones. +`--pause-redis` adds a controlled failure phase after the ordinary rounds. It +freezes only the owned Redis container, sends up to 64 concurrent renewal +requests, and requires HTTP 503 within an eight-second fixture bound. Snapshots +must also fail explicitly rather than return an empty room. Redis is unpaused +in `finally`; once reachable, higher-sequence renewals must recover the same +leases and exact membership. The test requires observed pool waiters and rejects +sampled active counts above the configured maximum. It does not retry failed +requests invisibly or raise production timeouts. + +The [Linux Redis-stall receipt](results/linux-redis-stall.json) passed with three +nodes, two tenants, 24 users and 96 tab leases. All 96 stalled renewals and all +three node snapshots returned the expected unavailable response. Existing +leases recovered after Redis resumed, and cleanup passed. This proves bounded +failure and recovery for an injected stall; it does not explain the intermittent +Windows timeout or establish a saturation throughput threshold. + ```powershell python docs/performance/presence_nodes.py --java java --node-image eclipse-temurin:21-jre --nodes 3 --tenants 2 --users 12 --tabs 4 --rounds 1000 ``` diff --git a/docs/performance/presence_nodes.py b/docs/performance/presence_nodes.py index 21d1b49..922030d 100644 --- a/docs/performance/presence_nodes.py +++ b/docs/performance/presence_nodes.py @@ -73,6 +73,7 @@ def main(): parser.add_argument("--workers", type=int, default=8) parser.add_argument("--crash-round", type=int, default=0, help="Kill the first node before this round; zero disables") parser.add_argument("--churn-every", type=int, default=0, help="Leave/rejoin the last user's tabs every N rounds; zero disables") + parser.add_argument("--pause-redis", action="store_true", help="After the workload, freeze owned Redis and verify bounded failures/recovery") args = parser.parse_args() for name, lower, upper in (("nodes", 2, 8), ("tenants", 2, 8), ("users", 2, 100), ("tabs", 2, 8), ("rounds", 1, 1000), ("workers", 1, 64)): @@ -216,6 +217,40 @@ def main(): result["churn_cycles"] = result.get("churn_cycles", 0) + 1 result["departed_leases_rejected"] = result.get("departed_leases_rejected", 0) + len(departing) result["completed_rounds"] = sequence + if args.pause_redis: + failures = [] + + def stalled_renew(item): + """A frozen authority must fail explicitly, never acknowledge a write.""" + index, (identity, record, tab, lease, user) = item + before = time.perf_counter() + response = request(nodes[index % len(nodes)], identity, "/presence/leases/renew", + {"connectionId": lease, "table": "person", "pk": record, + "mode": "EDITING" if user < 2 else "VIEWING", "sequence": args.rounds + 1}, + tab, expected=503) + if "viewers" in response or "temporarily unavailable" not in response.get("message", ""): + raise AssertionError("Stalled authority did not return the expected unavailable response") + duration = (time.perf_counter() - before) * 1000 + if duration > 8000: + raise AssertionError("Unavailable response exceeded the eight-second fixture bound") + return duration + + subprocess.run(["docker", "pause", container], check=True, capture_output=True) + try: + with ThreadPoolExecutor(max_workers=64) as burst: + failures.extend(burst.map(stalled_renew, enumerate(actors))) + for node in nodes: + response = request(node, "load-t0-u0", "/presence?table=person&pk=1", expected=503) + if "viewers" in response: + raise AssertionError("Stalled authority was presented as a room snapshot") + finally: + subprocess.run(["docker", "unpause", container], check=True, capture_output=True) + subprocess.run(["docker", "exec", container, "redis-cli", "PING"], check=True, capture_output=True, timeout=10) + for item in enumerate(actors): + renew(item, args.rounds + 2) + check_rooms() + result["redis_stall"] = {"rejected_renewals": len(failures), "failed_snapshot_nodes": len(nodes), + "response_times": distribution(failures), "same_leases_recovered": True} elapsed = time.perf_counter() - started result.update(status="PASS", seconds=elapsed, renewals=distribution(renewals), snapshots=distribution(snapshots), renewal_throughput=len(renewals) / elapsed, @@ -275,6 +310,8 @@ def main(): result["resources"] = resources if result["status"] == "PASS" and len(resources) != args.nodes: result.update(status="FAIL", error="Missing resource samples for a node") + if result["status"] == "PASS" and args.pause_redis and not any(node["poolWaiting_sampled_max"] > 0 for node in resources): + result.update(status="FAIL", error="Redis stall did not demonstrate pool waiters") cleanup = subprocess.run(["docker", "rm", "-f", container], capture_output=True) network_cleanup = True if network_created: diff --git a/docs/performance/results/linux-redis-stall.json b/docs/performance/results/linux-redis-stall.json new file mode 100644 index 0000000..652b748 --- /dev/null +++ b/docs/performance/results/linux-redis-stall.json @@ -0,0 +1,117 @@ +{ + "status": "PASS", + "parameters": { + "java": "java (Java 21)", + "node_image": "eclipse-temurin@sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99", + "nodes": 3, + "tenants": 2, + "users": 12, + "tabs": 4, + "rounds": 30, + "workers": 8, + "crash_round": 0, + "churn_every": 0, + "pause_redis": true + }, + "scope": "loopback HTTP, mock identities, memory records", + "node_image_id": "sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99", + "current_round": 30, + "completed_rounds": 30, + "redis_stall": { + "rejected_renewals": 96, + "failed_snapshot_nodes": 3, + "response_times": { + "samples": 96, + "p50_ms": 1013.8123999349773, + "p95_ms": 4014.4751999760047, + "p99_ms": 4015.4822999611497, + "max_ms": 4015.4822999611497 + }, + "same_leases_recovered": true + }, + "seconds": 20.33888840000145, + "renewals": { + "samples": 2880, + "p50_ms": 8.835799992084503, + "p95_ms": 23.678299970924854, + "p99_ms": 35.42319999542087, + "max_ms": 102.84149996004999 + }, + "snapshots": { + "samples": 372, + "p50_ms": 13.98350007366389, + "p95_ms": 16.946100047789514, + "p99_ms": 20.020700059831142, + "max_ms": 20.943500101566315 + }, + "renewal_throughput": 141.6006589622565, + "node_pids": [ + 88080, + 78036, + 121352 + ], + "pid_kind": "docker-client", + "node_containers": [ + "qqq-capacity-68ad627e7be5-node-58312", + "qqq-capacity-68ad627e7be5-node-58314", + "qqq-capacity-68ad627e7be5-node-58315" + ], + "redis_slow_commands": [ + { + "unix_time": 1793032016, + "duration_us": 2198, + "command": "EVALSHA" + } + ], + "redis_latency_events": [ + [ + "command", + 1793032016, + 2, + 2 + ] + ], + "resources": [ + { + "node_log": "node-58312.log", + "samples": 280, + "observed_wall_ms": 27899, + "cpu_seconds": 23.96, + "cpu_core_equivalents": 0.8588121438044375, + "heapUsedBytes_sampled_max": 164221400, + "heapCommittedBytes_sampled_max": 188743680, + "poolActive_sampled_max": 6, + "poolIdle_sampled_max": 5, + "poolWaiting_sampled_max": 15, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-58314.log", + "samples": 289, + "observed_wall_ms": 28800, + "cpu_seconds": 23.2, + "cpu_core_equivalents": 0.8055555555555556, + "heapUsedBytes_sampled_max": 162849016, + "heapCommittedBytes_sampled_max": 192937984, + "poolActive_sampled_max": 4, + "poolIdle_sampled_max": 3, + "poolWaiting_sampled_max": 14, + "poolMaximum_sampled_max": 8 + }, + { + "node_log": "node-58315.log", + "samples": 272, + "observed_wall_ms": 27100, + "cpu_seconds": 23.85, + "cpu_core_equivalents": 0.8800738007380073, + "heapUsedBytes_sampled_max": 165779696, + "heapCommittedBytes_sampled_max": 188743680, + "poolActive_sampled_max": 5, + "poolIdle_sampled_max": 4, + "poolWaiting_sampled_max": 14, + "poolMaximum_sampled_max": 8 + } + ], + "cleanup": true, + "receipt_note": "Java executable path normalized; measurements unchanged. Disposable fixture identities only." +} -- 2.54.0 From 56a8b760d9c1a1862cb11e80fad07cf5cdc20b3a Mon Sep 17 00:00:00 2001 From: James Maes Date: Thu, 29 Oct 2026 09:56:38 -0500 Subject: [PATCH 6/6] test(collaboration): capture multi-node presence capacity Co-Authored-By: Codex (GPT-6) --- docs/performance/MULTI-NODE.md | 93 ++++- docs/performance/browser_presence_latency.mjs | 69 +++- docs/performance/browser_presence_load.mjs | 75 ++++ .../browser_subscription_affinity.mjs | 60 +++ docs/performance/presence_nodes.py | 17 +- docs/performance/presence_router.mjs | 42 +++ .../results/browser-load-reconnect.json | 357 ++++++++++++++++++ .../results/browser-reconnect.json | 317 ++++++++++++++++ .../browser-subscription-affinity.json | 14 + .../results/linux-100-users-resources.json | 181 +++++++++ .../devserver/CapacityPresenceDevServer.java | 23 ++ 11 files changed, 1239 insertions(+), 9 deletions(-) create mode 100644 docs/performance/browser_presence_load.mjs create mode 100644 docs/performance/browser_subscription_affinity.mjs create mode 100644 docs/performance/presence_router.mjs create mode 100644 docs/performance/results/browser-load-reconnect.json create mode 100644 docs/performance/results/browser-reconnect.json create mode 100644 docs/performance/results/browser-subscription-affinity.json create mode 100644 docs/performance/results/linux-100-users-resources.json diff --git a/docs/performance/MULTI-NODE.md b/docs/performance/MULTI-NODE.md index 0445077..db3d1f0 100644 --- a/docs/performance/MULTI-NODE.md +++ b/docs/performance/MULTI-NODE.md @@ -32,10 +32,37 @@ used/committed bytes, active/idle/waiting pool counts and the configured maximum CPU core equivalents are process CPU time divided by observed wall time; they can exceed one for a multithreaded JVM. Observation begins after server startup and includes lease setup and workload execution. Unsupported CPU counters are -reported as null. Heap is not total resident memory. Sampled peaks may miss +reported as null. Linux fixtures also read process `VmRSS` from `/proc/self/status` +every 100 ms and record `residentBytes_sampled_max`; unsupported platforms report +null. Linux-container runs require resident-memory samples. Heap is not total +resident memory, and process RSS is not the container's complete cgroup footprint. +Sampled peaks may miss short bursts and must not be represented as exact maxima. Raw samples remain in node logs; missing node samples or a sampled pool-bound violation fail the run. +The final Redis INFO capture records `used_memory`, lifetime `used_memory_peak`, +and current `used_memory_rss`, plus process user/system CPU seconds since Redis +startup. Redis RSS is an end-of-run observation, not a sampled peak. These fresh, +owned Redis instances include fixture setup and diagnostics in their lifetime; +do not interpret those CPU values as workload-only measurements or add independent +process peaks and call the sum a measured simultaneous maximum. Missing Redis +memory/CPU diagnostics fail acceptance while cleanup still runs. + +The [100-user resource run](results/linux-100-users-resources.json) used three +Linux JVMs, two tenants with 50 people each, four leases per person, 16 request +workers and 200 rounds. All 80,000 renewals and 2,400 exact-membership snapshots +passed in 102.568 workload seconds. Renewal p50/p95/p99/max were +11.326/21.765/25.881/1,007.404 ms; snapshot p99 was 19.424 ms. The one-second +renewal maximum is retained, not discarded. Cleanup passed. + +Sampled JVM RSS peaks were 662,786,048 / 662,188,032 / 703,479,808 bytes +(approximately 632–671 MiB), while used-heap peaks were about 171–177 MiB. +CPU observations were 1.73–1.83 core equivalents per JVM; sampled pool-active +peaks were 7/7/6 of eight with no sampled waiters. Redis peak allocated memory +was 2,579,896 bytes; final RSS was 8,777,728 bytes. Redis process CPU since +startup was 23.738 system plus 35.526 user seconds. These measurements include +fixture overhead, omit browser load, and do not establish deployment limits. + Initial smoke run: three JVMs, two tenants, six users each, four tabs per user, three rounds: 144 renewals and 36 snapshots passed with tenant isolation, duplicate-tab deduplication and two editors retained. Cleanup passed. This short @@ -149,6 +176,17 @@ used for these runs was ## Browser propagation baseline +`browser_subscription_affinity.mjs FRONTEND_REPO BASE_URL RESULT_PATH` verifies +the current owning-node subscription contract against the two-node fixture. It +uses the connection ID from the real browser's successful subscription request, +then preserves the same session cookies while changing only the test routing +cookie. The [published-artifact receipt](results/browser-subscription-affinity.json) +records owner 200, other node 404, owner 200, with the requested channel active +on both successful responses and the unsaved draft retained. This proves why +subscription requests need owning-node routing even when presence state is shared. +It does not test the deployed Traefik cookie, HTTPS, buffering or failover. +The packaged runner accepts this script through `--affinity-script`. + `browser_presence_latency.mjs` measures Chromium field-focus changes until a different editor sees the corresponding remote marker. Supply a caller-owned two-node loopback fixture serving the published frontend with `alice`/`admin` @@ -159,12 +197,52 @@ affinity cookies: node docs/performance/browser_presence_latency.mjs /path/to/frontend http://localhost:PORT result.json 25 ``` +Use `presence_router.mjs FIRST_NODE_PORT SECOND_NODE_PORT ROUTER_PORT` for this +test. It binds only to loopback and adds a test-only POST endpoint that severs +first-node SSE responses. Never deploy this fault-injection router. The packaged +acceptance fixture accepts it through `--router-script` alongside `--latency-script`. + The test verifies the target marker is absent before every change, the old marker disappears afterward, both real SSE streams opened successfully, issued-lease writes reached distinct nodes, and unsaved drafts survive. API responses are not mocked. The fixture owns backend/router/Redis cleanup; this script owns and closes its browser. +After timing the normal transitions, the script takes the observer offline and +explicitly severs its existing SSE response through the fixture router. It requires +an observed stream failure, a failed offline fetch, three acknowledged sender focus +changes, and absence of the new marker while disconnected. After network restoration +it requires a new successful SSE response and the current marker with the previous +marker removed, without reload or loss of either unsaved draft. Recovery may use +the shipped reconnect and polling behavior; this does not prove event replay. + +The [disconnect/recovery receipt](results/browser-reconnect.json) passed against +the public RC37 JAR: one stream severed, three missed changes, recovery in 1,858 ms, +both drafts preserved. The accompanying six-user and Redis admin checks passed; +fixture cleanup passed. This is one Chromium disconnect case without concurrent +load, not a reconnect storm, lease-expiry test, or production recovery objective. + +Append `load` to the browser command (or use `--latency-load` on the packaged +fixture) to add 16 concurrent HTTP lease workers: four mock identities with four +leases each on the same record, split between both nodes. Each worker renews +sequentially with a 100 ms pause after each response. The browser observations +are spread over at least 60 seconds; every observation interval must include +completed background renewals. The workload continues during disconnect/reconnect. +Any request or lease-cleanup failure fails the test. Responses are disposed after +reading, and workers stop before the contexts close. The receipt reports completed +renewals, observed node counts, elapsed time and any errors. These are HTTP load +workers, not 16 additional browser tabs or distinct people. + +The [bounded load/reconnect receipt](results/browser-load-reconnect.json) passed +9,046 renewals over 69.728 seconds, with no worker or lease-release errors. +Across 25 observed focus transitions, p50/p95/p99 were 32.384/85.391/785.495 ms; +p99 is the maximum of this small sample, not a reliable tail-latency estimate. +The observer recovered in 1,855.887 ms after missing three updates while load +continued. Both drafts survived. Six-user and Redis admin acceptance also passed +against the public RC37 JAR; fixture cleanup passed. An initial run failed only +because the new load helper used the wrong lease-release endpoint; that receipt +is retained in the local packaged fixture history, not counted as a passing run. + The [25-transition receipt](results/browser-focus-25.json) passed against the downloaded public RC37 JAR and published backend snapshots. Focus-to-observed-DOM p50/p95/p99 were 31.648/33.078/36.092 ms; p99 is the maximum in this small sample. @@ -173,11 +251,14 @@ admin browser checks also passed in the same fixture run; fixture cleanup passed This baseline includes browser automation, UI debounce, network, invalidation and rendering. It is not a server-only transport measurement, an SSE-only causal -proof, an under-load result, or a production latency objective. Browser propagation -under sustained load, churn and failures remains unqualified. +proof, an under-load result, or a production latency objective. The separate load +receipt above covers one bounded workload and disconnect scenario; larger loads, +reconnect storms and deployment-specific behavior remain unqualified. -Remaining qualification includes larger sustained workloads, total resident -memory and Redis-server resource measurements, browser/SSE propagation percentiles, saturation, churn, -missed messages and process crashes. Loopback HTTP latency must not be reported +Remaining qualification includes longer soak and larger workloads, container +resource limits, reconnect storms and the actual deployment's capacity and alert +thresholds. The receipts above cover specific bounded HTTP, browser, saturation, +churn, missed-update and process-crash cases, not every combination. The native +Windows intermittent Redis read timeout remains unresolved. Loopback HTTP latency must not be reported as browser propagation or a production SLA. No deployed capacity threshold is established by this fixture. diff --git a/docs/performance/browser_presence_latency.mjs b/docs/performance/browser_presence_latency.mjs index b3a6190..be5944b 100644 --- a/docs/performance/browser_presence_latency.mjs +++ b/docs/performance/browser_presence_latency.mjs @@ -3,8 +3,11 @@ import { createRequire } from 'node:module'; import { resolve } from 'node:path'; import { writeFile } from 'node:fs/promises'; import { performance } from 'node:perf_hooks'; +import { setTimeout as delay } from 'node:timers/promises'; +import { startPresenceLoad } from './browser_presence_load.mjs'; -const [frontendRepo, base, output, countValue = '25'] = process.argv.slice(2); +const [frontendRepo, base, output, countValue = '25', loadMode] = process.argv.slice(2); +if (loadMode && loadMode !== 'load') throw new Error('Optional final argument must be load'); const samples = Number(countValue); if (!frontendRepo || !base || !output || !Number.isInteger(samples) || samples < 5 || samples > 100) { throw new Error('Expected frontend repo, loopback base URL, result path and 5..100 samples'); @@ -17,6 +20,7 @@ const { chromium, expect } = require('@playwright/test'); const result = { status: 'FAIL', samples: [], streams: [], requests: [], errors: [], scope: 'Chromium focus-to-remote-DOM observation, including automation and UI debounce; no concurrent load' }; const browser = await chromium.launch({ headless: true }); +let load; try { async function editor(user, node) { const context = await browser.newContext(); @@ -45,7 +49,14 @@ try { await observer.locator('[data-qqq-id="firstName"]').fill('Retained observer draft'); await sender.locator('[data-qqq-id="firstName"]').fill('Retained sender draft'); await expect(observer.locator('[data-presence-field="firstName"]')).toContainText('Demo Administrator'); + if (loadMode) { + load = await startPresenceLoad(browser, base, expect); + result.load = load.state; + result.scope = 'Chromium focus-to-remote-DOM and reconnect under 16 concurrent HTTP lease workers; loopback, four background identities'; + } for (let index = 0; index < samples; index++) { + const previousCompleted = load?.state.completed; + if (load) await delay(60000 / samples); const field = index % 2 === 0 ? 'lastName' : 'firstName'; const previous = field === 'firstName' ? 'lastName' : 'firstName'; await expect(observer.locator(`[data-presence-field="${field}"]`)).not.toContainText('Demo Administrator'); @@ -54,6 +65,61 @@ try { await expect(observer.locator(`[data-presence-field="${field}"]`)).toContainText('Demo Administrator'); result.samples.push({ field, propagation_ms: performance.now() - before }); await expect(observer.locator(`[data-presence-field="${previous}"]`)).not.toContainText('Demo Administrator'); + if (load) { + expect(load.state.errors).toEqual([]); + expect(load.state.completed).toBeGreaterThan(previousCompleted); + } + } + // Drop a real browser connection, miss multiple focus updates, then converge + // through the shipped reconnect/polling behavior without reloading the form. + const oldField = samples % 2 === 0 ? 'firstName' : 'lastName'; + const newField = oldField === 'firstName' ? 'lastName' : 'firstName'; + const streamsBefore = result.streams.filter(stream => stream.user === 'alice').length; + result.reconnect = { status: 'FAIL', missed_changes: 0 }; + const interrupted = observer.waitForEvent('requestfailed', { + predicate: request => new URL(request.url()).pathname.endsWith('/realtime/stream'), + timeout: 10000, + }); + await observer.context().setOffline(true); + const disconnect = await fetch(`${base}/__acceptance/disconnect-first`, { method: 'POST' }); + expect(disconnect.status).toBe(200); + result.reconnect.disconnected_streams = (await disconnect.json()).disconnected; + expect(result.reconnect.disconnected_streams).toBeGreaterThan(0); + const failedStream = await interrupted; + result.reconnect.stream_failure = failedStream.failure()?.errorText; + const offlineFetch = await observer.evaluate(async () => { + try { + await fetch(`/app/?offlineProbe=${Date.now()}`, { cache: 'no-store' }); + return false; + } catch { return true; } + }); + expect(offlineFetch).toBe(true); + for (const field of [newField, oldField, newField]) { + const renewed = sender.waitForResponse(response => + new URL(response.url()).pathname.endsWith('/presence/leases/renew') && + response.request().method() === 'POST' && response.status() === 200 && + response.request().postDataJSON()?.fieldName === field); + await sender.locator(`[data-qqq-id="${field}"]`).focus(); + await renewed; + result.reconnect.missed_changes++; + } + await expect(observer.locator(`[data-presence-field="${newField}"]`)).not.toContainText('Demo Administrator'); + const resumed = performance.now(); + await observer.context().setOffline(false); + await expect.poll(() => result.streams.filter(stream => stream.user === 'alice' && + stream.status === 200 && stream.type?.includes('text/event-stream')).length, + { timeout: 30000 }).toBeGreaterThan(streamsBefore); + await expect(observer.locator(`[data-presence-field="${newField}"]`)).toContainText('Demo Administrator', { timeout: 30000 }); + await expect(observer.locator(`[data-presence-field="${oldField}"]`)).not.toContainText('Demo Administrator'); + result.reconnect.recovery_ms = performance.now() - resumed; + result.reconnect.status = 'PASS'; + if (load) { + await load.stop(); + expect(load.state.errors).toEqual([]); + expect(load.state.elapsed_ms).toBeGreaterThanOrEqual(60000); + expect(load.state.nodes.first).toBeGreaterThan(0); + expect(load.state.nodes.second).toBeGreaterThan(0); + load = undefined; } await expect(observer.locator('[data-qqq-id="firstName"]')).toHaveValue('Retained observer draft'); await expect(sender.locator('[data-qqq-id="firstName"]')).toHaveValue('Retained sender draft'); @@ -72,6 +138,7 @@ try { result.failure = String(error); process.exitCode = 1; } finally { + if (load) await load.stop(); await browser.close(); await writeFile(output, JSON.stringify(result, null, 2)); console.log(JSON.stringify(result, null, 2)); diff --git a/docs/performance/browser_presence_load.mjs b/docs/performance/browser_presence_load.mjs new file mode 100644 index 0000000..562f0c0 --- /dev/null +++ b/docs/performance/browser_presence_load.mjs @@ -0,0 +1,75 @@ +// Bounded real HTTP load using caller-owned authenticated browser contexts. +import { performance } from 'node:perf_hooks'; +import { setTimeout as delay } from 'node:timers/promises'; + +export async function startPresenceLoad(browser, base, expect) { + const state = { workers: 16, completed: 0, errors: [], nodes: {}, elapsed_ms: 0 }; + const contexts = []; + const leases = []; + let running = true; + let stopped = false; + let tasks = []; + const began = performance.now(); + const endpoint = `${base}/qqq/v1/qbit/collaboration/presence/leases`; + async function post(context, path, data) { + const response = await context.request.post(endpoint + path, { data, timeout: 10000 }); + try { + expect(response.status()).toBe(200); + const node = response.headers()['x-acceptance-node']; + state.nodes[node] = (state.nodes[node] ?? 0) + 1; + return await response.json(); + } finally { await response.dispose(); } + } + async function stop() { + if (stopped) return; + stopped = true; + running = false; + await Promise.all(tasks); + state.elapsed_ms = performance.now() - began; + for (const lease of leases) { + try { + const response = await lease.context.request.delete(`${base}/qqq/v1/qbit/collaboration/presence`, { + data: { ...lease.target, connectionId: lease.id, sequence: ++lease.sequence }, timeout: 10000, + }); + try { expect(response.status()).toBe(200); } finally { await response.dispose(); } + } catch (error) { state.errors.push(String(error)); } + } + for (const context of contexts) await context.close(); + } + try { + for (const [index, user] of ['bob', 'carol', 'dave', 'erin'].entries()) { + const context = await browser.newContext(); + contexts.push(context); + await context.addCookies([ + { name: 'collab_dev_user', value: user, url: base }, + { name: 'presence_node', value: index % 2 ? 'second' : 'first', url: base }, + ]); + const page = await context.newPage(); + await page.goto(`${base}/app/person/1?createMockSession=true`); + await expect(page.locator('[data-qqq-id="record-view-title-person"]')).toBeVisible(); + await page.close(); + for (let tab = 0; tab < 4; tab++) { + const target = { table: 'person', pk: '1' }; + const opened = await post(context, '', target); + expect(typeof opened.connectionId).toBe('string'); + leases.push({ context, target, id: opened.connectionId, sequence: 0, completed: 0 }); + } + } + tasks = leases.map(async lease => { + try { + while (running) { + await post(lease.context, '/renew', { ...lease.target, connectionId: lease.id, + sequence: ++lease.sequence, mode: 'VIEWING' }); + lease.completed++; + state.completed++; + await delay(100); + } + } catch (error) { state.errors.push(String(error)); running = false; } + }); + await expect.poll(() => leases.filter(lease => lease.completed > 0).length).toBe(16); + return { state, stop }; + } catch (error) { + await stop(); + throw error; + } +} diff --git a/docs/performance/browser_subscription_affinity.mjs b/docs/performance/browser_subscription_affinity.mjs new file mode 100644 index 0000000..374f466 --- /dev/null +++ b/docs/performance/browser_subscription_affinity.mjs @@ -0,0 +1,60 @@ +// Verify owning-node routing with real published UI and backend artifacts. +import { createRequire } from 'node:module'; +import { resolve } from 'node:path'; +import { writeFile } from 'node:fs/promises'; + +const [frontendRepo, base, output] = process.argv.slice(2); +if (!frontendRepo || !output || !['localhost', '127.0.0.1'].includes(new URL(base).hostname)) { + throw new Error('Expected frontend repo, owned loopback URL and result path'); +} +const require = createRequire(resolve(frontendRepo, 'package.json')); +const { chromium, expect } = require('@playwright/test'); +const result = { status: 'FAIL', checks: [], errors: [] }; +const browser = await chromium.launch({ headless: true }); +try { + const context = await browser.newContext(); + await context.addCookies([ + { name: 'collab_dev_user', value: 'alice', url: base }, + { name: 'presence_node', value: 'first', url: base }, + ]); + const page = await context.newPage(); + page.on('pageerror', error => result.errors.push(error.message)); + const subscribed = page.waitForResponse(response => + new URL(response.url()).pathname.endsWith('/realtime/subscriptions') && response.status() === 200); + await page.goto(`${base}/app/person/1/edit?createMockSession=true`); + await expect(page.locator('[data-qqq-id="firstName"]')).toBeVisible(); + await page.locator('[data-qqq-id="firstName"]').fill('Keep this unsaved draft'); + const response = await subscribed; + expect(response.headers()['x-acceptance-node']).toBe('first'); + const { connectionId } = response.request().postDataJSON(); + expect(typeof connectionId).toBe('string'); + for (const [node, expected] of [['first', 200], ['second', 404], ['first', 200]]) { + // Keep the same session/user cookies and change only the test routing cookie. + const cookies = (await context.cookies(base)).map(cookie => + `${cookie.name}=${cookie.name === 'presence_node' ? node : cookie.value}`).join('; '); + const changed = await context.request.post(`${base}/qqq/v1/realtime/subscriptions`, { + headers: { Cookie: cookies }, + data: { connectionId, add: ['record:person:1'], remove: [] }, timeout: 10000, + }); + try { + result.checks.push({ node: changed.headers()['x-acceptance-node'], status: changed.status() }); + expect(changed.headers()['x-acceptance-node']).toBe(node); + expect(changed.status()).toBe(expected); + if (expected === 200) { + const body = await changed.json(); + expect(body.active).toContain('record:person:1'); + expect(body.refused ?? []).toEqual([]); + } + } finally { await changed.dispose(); } + } + await expect(page.locator('[data-qqq-id="firstName"]')).toHaveValue('Keep this unsaved draft'); + expect(result.errors).toEqual([]); + result.status = 'PASS'; +} catch (error) { + result.failure = String(error); + process.exitCode = 1; +} finally { + await browser.close(); + await writeFile(output, JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); +} diff --git a/docs/performance/presence_nodes.py b/docs/performance/presence_nodes.py index 922030d..6ab098a 100644 --- a/docs/performance/presence_nodes.py +++ b/docs/performance/presence_nodes.py @@ -271,7 +271,14 @@ def main(): for section in ("cpu", "memory", "commandstats", "stats"): value = subprocess.check_output(["docker", "exec", container, "redis-cli", "INFO", section], text=True, timeout=10) (output / f"redis-{section}.txt").write_text(value) - except (subprocess.SubprocessError, ValueError) as error: + parsed = dict(line.split(":", 1) for line in value.splitlines() if ":" in line and not line.startswith("#")) + if section == "memory": + result["redis_memory"] = {key: int(parsed[key]) for key in + ("used_memory", "used_memory_peak", "used_memory_rss")} + if section == "cpu": + result["redis_cpu_seconds_since_start"] = {key: float(parsed[key]) for key in + ("used_cpu_sys", "used_cpu_user")} + except (subprocess.SubprocessError, ValueError, KeyError) as error: result["redis_diagnostic_error"] = type(error).__name__ node_cleanup = True for node_container in node_containers: @@ -304,10 +311,16 @@ def main(): "cpu_seconds": cpu_ns / 1e9 if first["cpuNanos"] >= 0 else None, "cpu_core_equivalents": cpu_ns / (wall_ms * 1e6) if wall_ms > 0 and first["cpuNanos"] >= 0 else None, **{key + "_sampled_max": max(sample[key] for sample in samples) - for key in ("heapUsedBytes", "heapCommittedBytes", "poolActive", "poolIdle", "poolWaiting", "poolMaximum")}}) + for key in ("heapUsedBytes", "heapCommittedBytes", "poolActive", "poolIdle", "poolWaiting", "poolMaximum")}, + "residentBytes_sampled_max": max(sample["residentBytes"] for sample in samples) + if all(sample.get("residentBytes", -1) >= 0 for sample in samples) else None}) if any(sample["poolActive"] > sample["poolMaximum"] for sample in samples): result.update(status="FAIL", error="Sampled pool active count exceeded configured maximum") result["resources"] = resources + if result["status"] == "PASS" and args.node_image and any(node["residentBytes_sampled_max"] is None for node in resources): + result.update(status="FAIL", error="Linux resident-memory samples missing") + if result["status"] == "PASS" and any(key not in result for key in ("redis_memory", "redis_cpu_seconds_since_start")): + result.update(status="FAIL", error="Redis memory/CPU diagnostics missing") if result["status"] == "PASS" and len(resources) != args.nodes: result.update(status="FAIL", error="Missing resource samples for a node") if result["status"] == "PASS" and args.pause_redis and not any(node["poolWaiting_sampled_max"] > 0 for node in resources): diff --git a/docs/performance/presence_router.mjs b/docs/performance/presence_router.mjs new file mode 100644 index 0000000..e840b0b --- /dev/null +++ b/docs/performance/presence_router.mjs @@ -0,0 +1,42 @@ +// Disposable loopback fixture only; never deploy this fault-injection router. +import http from 'node:http'; + +const [first, second, listen] = process.argv.slice(2).map(Number); +if (![first, second, listen].every(port => Number.isInteger(port) && port > 0 && port < 65536)) { + throw new Error('Expected first-node, second-node and router ports'); +} +const firstStreams = new Set(); +const server = http.createServer((request, response) => { + if (request.method === 'POST' && request.url === '/__acceptance/disconnect-first') { + const disconnected = firstStreams.size; + for (const stream of firstStreams) stream.destroy(); + response.writeHead(200, { 'content-type': 'application/json' }); + response.end(JSON.stringify({ disconnected })); + return; + } + const isSecond = /(?:^|;\s*)presence_node=second(?:;|$)/.test(request.headers.cookie ?? ''); + const port = isSecond ? second : first; + const upstream = http.request({ + hostname: '127.0.0.1', port, path: request.url, method: request.method, + headers: { ...request.headers, host: `127.0.0.1:${port}` }, + }, incoming => { + response.writeHead(incoming.statusCode, { + ...incoming.headers, 'x-acceptance-node': isSecond ? 'second' : 'first', + }); + if (!isSecond && incoming.headers['content-type']?.includes('text/event-stream')) { + firstStreams.add(response); + } + incoming.pipe(response); + }); + upstream.on('error', () => { + if (!response.headersSent) response.writeHead(502); + response.end('Acceptance node unavailable'); + }); + response.on('close', () => { + firstStreams.delete(response); + upstream.destroy(); + }); + request.pipe(upstream); +}); +server.listen(listen, '127.0.0.1', () => console.log(`PRESENCE_ROUTER_READY ${listen}`)); +for (const signal of ['SIGINT', 'SIGTERM']) process.on(signal, () => server.close()); diff --git a/docs/performance/results/browser-load-reconnect.json b/docs/performance/results/browser-load-reconnect.json new file mode 100644 index 0000000..b2eb0cb --- /dev/null +++ b/docs/performance/results/browser-load-reconnect.json @@ -0,0 +1,357 @@ +{ + "status": "PASS", + "samples": [ + { + "field": "lastName", + "propagation_ms": 36.560600000000704 + }, + { + "field": "firstName", + "propagation_ms": 33.44050000000061 + }, + { + "field": "lastName", + "propagation_ms": 32.50890000000072 + }, + { + "field": "firstName", + "propagation_ms": 785.4951000000001 + }, + { + "field": "lastName", + "propagation_ms": 85.39069999999992 + }, + { + "field": "firstName", + "propagation_ms": 30.589599999999336 + }, + { + "field": "lastName", + "propagation_ms": 32.756099999998696 + }, + { + "field": "firstName", + "propagation_ms": 38.17399999999907 + }, + { + "field": "lastName", + "propagation_ms": 30.243099999999686 + }, + { + "field": "firstName", + "propagation_ms": 29.600799999992887 + }, + { + "field": "lastName", + "propagation_ms": 29.57220000000234 + }, + { + "field": "firstName", + "propagation_ms": 29.197800000001735 + }, + { + "field": "lastName", + "propagation_ms": 33.05130000000645 + }, + { + "field": "firstName", + "propagation_ms": 32.38390000000072 + }, + { + "field": "lastName", + "propagation_ms": 31.52850000000035 + }, + { + "field": "firstName", + "propagation_ms": 34.66169999999693 + }, + { + "field": "lastName", + "propagation_ms": 31.258300000001327 + }, + { + "field": "firstName", + "propagation_ms": 44.19029999999475 + }, + { + "field": "lastName", + "propagation_ms": 36.56189999999333 + }, + { + "field": "firstName", + "propagation_ms": 30.240600000004633 + }, + { + "field": "lastName", + "propagation_ms": 29.79899999999907 + }, + { + "field": "firstName", + "propagation_ms": 29.621500000001106 + }, + { + "field": "lastName", + "propagation_ms": 31.66189999999915 + }, + { + "field": "firstName", + "propagation_ms": 29.198399999993853 + }, + { + "field": "lastName", + "propagation_ms": 32.931200000006356 + } + ], + "streams": [ + { + "user": "alice", + "status": 200, + "type": "text/event-stream;charset=utf-8" + }, + { + "user": "admin", + "status": 200, + "type": "text/event-stream;charset=utf-8" + }, + { + "user": "alice", + "status": 200, + "type": "text/event-stream;charset=utf-8" + } + ], + "requests": [ + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + } + ], + "errors": [], + "scope": "Chromium focus-to-remote-DOM and reconnect under 16 concurrent HTTP lease workers; loopback, four background identities", + "load": { + "workers": 16, + "completed": 9046, + "errors": [], + "nodes": { + "first": 4540, + "second": 4522 + }, + "elapsed_ms": 69728.3579 + }, + "reconnect": { + "status": "PASS", + "missed_changes": 3, + "disconnected_streams": 1, + "stream_failure": "net::ERR_INCOMPLETE_CHUNKED_ENCODING", + "recovery_ms": 1855.8868999999977 + }, + "percentiles_ms": { + "p50": 32.38390000000072, + "p95": 85.39069999999992, + "p99": 785.4951000000001, + "max": 785.4951000000001 + }, + "frontend_jar_sha256": "c9fa2eb2381a47c8104f82f4e940d2bfa8ba67e2341298dde0de4e21ad3ff1ab", + "fixture_cleanup": true, + "backend_source": "Published core cc3981bb3/collaboration ecff347/Redis7fe5492 snapshots; two independent JVMs, mock identities and memory records.", + "fixture_checks": [ + "browser-six-presence", + "browser-redis-admin", + "browser-presence-latency" + ] +} diff --git a/docs/performance/results/browser-reconnect.json b/docs/performance/results/browser-reconnect.json new file mode 100644 index 0000000..16d4616 --- /dev/null +++ b/docs/performance/results/browser-reconnect.json @@ -0,0 +1,317 @@ +{ + "status": "PASS", + "samples": [ + { + "field": "lastName", + "propagation_ms": 37.25770000000011 + }, + { + "field": "firstName", + "propagation_ms": 32.14499999999998 + }, + { + "field": "lastName", + "propagation_ms": 32.40950000000021 + }, + { + "field": "firstName", + "propagation_ms": 33.52500000000009 + }, + { + "field": "lastName", + "propagation_ms": 31.228200000000015 + }, + { + "field": "firstName", + "propagation_ms": 30.430999999999585 + }, + { + "field": "lastName", + "propagation_ms": 31.493199999999888 + }, + { + "field": "firstName", + "propagation_ms": 33.14539999999988 + }, + { + "field": "lastName", + "propagation_ms": 32.11259999999993 + }, + { + "field": "firstName", + "propagation_ms": 33.23129999999992 + }, + { + "field": "lastName", + "propagation_ms": 31.06800000000021 + }, + { + "field": "firstName", + "propagation_ms": 30.109199999999873 + }, + { + "field": "lastName", + "propagation_ms": 31.534299999999803 + }, + { + "field": "firstName", + "propagation_ms": 33.770800000000236 + }, + { + "field": "lastName", + "propagation_ms": 31.654199999999946 + }, + { + "field": "firstName", + "propagation_ms": 33.31700000000001 + }, + { + "field": "lastName", + "propagation_ms": 33.108600000000024 + }, + { + "field": "firstName", + "propagation_ms": 32.27370000000019 + }, + { + "field": "lastName", + "propagation_ms": 31.452499999999873 + }, + { + "field": "firstName", + "propagation_ms": 31.86220000000003 + }, + { + "field": "lastName", + "propagation_ms": 31.734399999999823 + }, + { + "field": "firstName", + "propagation_ms": 32.04489999999987 + }, + { + "field": "lastName", + "propagation_ms": 31.051899999999932 + }, + { + "field": "firstName", + "propagation_ms": 30.876300000000356 + }, + { + "field": "lastName", + "propagation_ms": 31.56970000000001 + } + ], + "streams": [ + { + "user": "alice", + "status": 200, + "type": "text/event-stream;charset=utf-8" + }, + { + "user": "admin", + "status": 200, + "type": "text/event-stream;charset=utf-8" + }, + { + "user": "alice", + "status": 200, + "type": "text/event-stream;charset=utf-8" + } + ], + "requests": [ + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "alice", + "status": 200, + "node": "first" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + }, + { + "user": "admin", + "status": 200, + "node": "second" + } + ], + "errors": [], + "scope": "Chromium focus-to-remote-DOM observation, including automation and UI debounce; no concurrent load", + "reconnect": { + "status": "PASS", + "missed_changes": 3, + "disconnected_streams": 1, + "stream_failure": "net::ERR_INCOMPLETE_CHUNKED_ENCODING", + "recovery_ms": 1858.2972000000004 + }, + "percentiles_ms": { + "p50": 31.86220000000003, + "p95": 33.770800000000236, + "p99": 37.25770000000011, + "max": 37.25770000000011 + }, + "frontend_jar_sha256": "c9fa2eb2381a47c8104f82f4e940d2bfa8ba67e2341298dde0de4e21ad3ff1ab", + "fixture_cleanup": true, + "backend_source": "Published core cc3981bb3/collaboration ecff347/Redis7fe5492 snapshots; two independent JVMs, mock identities and memory records.", + "fixture_checks": [ + "browser-six-presence", + "browser-redis-admin", + "browser-presence-latency" + ] +} diff --git a/docs/performance/results/browser-subscription-affinity.json b/docs/performance/results/browser-subscription-affinity.json new file mode 100644 index 0000000..69840c8 --- /dev/null +++ b/docs/performance/results/browser-subscription-affinity.json @@ -0,0 +1,14 @@ +{ + "status": "PASS", + "checks": [ + { "node": "first", "status": 200 }, + { "node": "second", "status": 404 }, + { "node": "first", "status": 200 } + ], + "errors": [], + "fixture_cleanup": true, + "fixture_checks": ["browser-six-presence", "browser-redis-admin", "browser-subscription-affinity"], + "frontend_jar_sha256": "c9fa2eb2381a47c8104f82f4e940d2bfa8ba67e2341298dde0de4e21ad3ff1ab", + "backend_source": "Published core cc3981bb3/collaboration ecff347/Redis7fe5492 snapshots; two independent JVMs, mock identities and memory records.", + "scope": "Same browser session and live SSE connection ID; change only test-router affinity. Not deployed Traefik qualification." +} diff --git a/docs/performance/results/linux-100-users-resources.json b/docs/performance/results/linux-100-users-resources.json new file mode 100644 index 0000000..248f02a --- /dev/null +++ b/docs/performance/results/linux-100-users-resources.json @@ -0,0 +1,181 @@ +{ + "status": "PASS", + "parameters": { + "java": "/path/to/java21", + "node_image": "eclipse-temurin@sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99", + "nodes": 3, + "tenants": 2, + "users": 50, + "tabs": 4, + "rounds": 200, + "workers": 16, + "crash_round": 0, + "churn_every": 0, + "pause_redis": false + }, + "scope": "loopback HTTP, mock identities, memory records", + "node_image_id": "sha256:227118764cccd3c3294617061aaa93a08309c63a395eefa83e6b35032992ea99", + "current_round": 200, + "completed_rounds": 200, + "seconds": 102.56801009993069, + "renewals": { + "samples": 80000, + "p50_ms": 11.325599974952638, + "p95_ms": 21.764500066637993, + "p99_ms": 25.881099980324507, + "max_ms": 1007.4040000326931 + }, + "snapshots": { + "samples": 2400, + "p50_ms": 15.130399959161878, + "p95_ms": 16.613700077868998, + "p99_ms": 19.423600053414702, + "max_ms": 28.864299994893372 + }, + "renewal_throughput": 779.9702843221492, + "node_pids": [ + 105696, + 4828, + 117588 + ], + "pid_kind": "docker-client", + "node_containers": [ + "qqq-capacity-f58f3ddcf5f3-node-59675", + "qqq-capacity-f58f3ddcf5f3-node-59676", + "qqq-capacity-f58f3ddcf5f3-node-59677" + ], + "redis_slow_commands": [ + { + "unix_time": 1793033347, + "duration_us": 1432, + "command": "EVALSHA" + }, + { + "unix_time": 1793033347, + "duration_us": 1612, + "command": "EVALSHA" + }, + { + "unix_time": 1793033340, + "duration_us": 1877, + "command": "EVALSHA" + }, + { + "unix_time": 1793033340, + "duration_us": 1313, + "command": "ZRANGE" + }, + { + "unix_time": 1793033336, + "duration_us": 1316, + "command": "EVALSHA" + }, + { + "unix_time": 1793033305, + "duration_us": 1459, + "command": "EVALSHA" + }, + { + "unix_time": 1793033300, + "duration_us": 1767, + "command": "EVALSHA" + }, + { + "unix_time": 1793033290, + "duration_us": 1180, + "command": "EVALSHA" + }, + { + "unix_time": 1793033283, + "duration_us": 1421, + "command": "EVALSHA" + }, + { + "unix_time": 1793033260, + "duration_us": 1291, + "command": "EVALSHA" + }, + { + "unix_time": 1793033260, + "duration_us": 1551, + "command": "EVALSHA" + }, + { + "unix_time": 1793033260, + "duration_us": 1083, + "command": "ZRANGE" + }, + { + "unix_time": 1793033255, + "duration_us": 1039, + "command": "EVALSHA" + }, + { + "unix_time": 1793033254, + "duration_us": 1215, + "command": "EVALSHA" + } + ], + "redis_latency_events": [ + [ + "command", + 1793033347, + 1, + 1 + ] + ], + "redis_cpu_seconds_since_start": { + "used_cpu_sys": 23.737706, + "used_cpu_user": 35.526214 + }, + "redis_memory": { + "used_memory": 2415952, + "used_memory_peak": 2579896, + "used_memory_rss": 8777728 + }, + "resources": [ + { + "node_log": "node-59675.log", + "samples": 1112, + "observed_wall_ms": 111099, + "cpu_seconds": 203.14, + "cpu_core_equivalents": 1.828459302063925, + "heapUsedBytes_sampled_max": 181098896, + "heapCommittedBytes_sampled_max": 211812352, + "poolActive_sampled_max": 7, + "poolIdle_sampled_max": 7, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8, + "residentBytes_sampled_max": 662786048 + }, + { + "node_log": "node-59676.log", + "samples": 1145, + "observed_wall_ms": 114399, + "cpu_seconds": 200.44, + "cpu_core_equivalents": 1.7521132177728826, + "heapUsedBytes_sampled_max": 185167416, + "heapCommittedBytes_sampled_max": 219152384, + "poolActive_sampled_max": 7, + "poolIdle_sampled_max": 7, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8, + "residentBytes_sampled_max": 662188032 + }, + { + "node_log": "node-59677.log", + "samples": 1159, + "observed_wall_ms": 115800, + "cpu_seconds": 200.16, + "cpu_core_equivalents": 1.728497409326425, + "heapUsedBytes_sampled_max": 179378632, + "heapCommittedBytes_sampled_max": 210763776, + "poolActive_sampled_max": 6, + "poolIdle_sampled_max": 7, + "poolWaiting_sampled_max": 0, + "poolMaximum_sampled_max": 8, + "residentBytes_sampled_max": 703479808 + } + ], + "cleanup": true +} diff --git a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java index 19bb859..1d2752d 100644 --- a/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java +++ b/src/test/java/com/kingsrook/qbits/collaboration/devserver/CapacityPresenceDevServer.java @@ -87,6 +87,7 @@ public class CapacityPresenceDevServer values.put("cpuNanos", ProcessHandle.current().info().totalCpuDuration().map(java.time.Duration::toNanos).orElse(-1L)); values.put("heapUsedBytes", heap.getUsed()); values.put("heapCommittedBytes", heap.getCommitted()); + values.put("residentBytes", residentBytes()); values.put("poolActive", pool.active()); values.put("poolIdle", pool.idle()); values.put("poolWaiting", pool.waiting()); @@ -109,6 +110,28 @@ public class CapacityPresenceDevServer } } + /*************************************************************************** + ** Read Linux process resident memory; unsupported platforms report -1. + ***************************************************************************/ + private static long residentBytes() + { + try + { + for(String line : java.nio.file.Files.readAllLines(java.nio.file.Path.of("/proc/self/status"))) + { + if(line.startsWith("VmRSS:")) + { + return (Long.parseLong(line.trim().split("\\s+")[1]) * 1024L); + } + } + } + catch(java.io.IOException | NumberFormatException e) + { + return (-1L); + } + return (-1L); + } + /*************************************************************************** ** Tenant comes from the configured mock identity, never a request tenant parameter. ***************************************************************************/ -- 2.54.0