fix(session): keep the load balancer's affinity cookies so requests and the realtime socket reach one node

A QQQ server that runs as several nodes sits behind a load balancer that names
the node it chose in a cookie. The session client attaches its cookies by hand
and dropped everything but the session, so the realtime socket could open on
one node while claims and downloads went to another; a server whose claim gate
needs the socket on the same node (qbit-printing 1.1.x) then answers 409.

- Keep the Application Load Balancer's cookies (names beginning AWSALB) from
  any answer of the session origin, the WebSocket upgrade answer included,
  whether it was accepted or refused. Send them on every later request to that
  origin and on the upgrade. Replace or drop them when the load balancer
  rewrites or expires them. In memory only; never saved with the session.
- Only cookies scoped to this host and the root path with a plain, bounded
  value are kept; at most eight; a Secure cookie only over HTTPS.
- Cookies under names other than the session's no longer decide whether a
  session answer is valid: a load balancer adds its own to every answer.

Tests cover the cookie rules, rotation and expiry, the upgrade carrying and
adopting the cookie, and what a refused request means when a node stops: a 5xx
or a dropped connection keeps the session, only a 401 ends it.

Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Signed-off-by: James Maes <james@kof22.com>
This commit is contained in:
james.maesandClaude Fable 5.2 committed 2026-11-10 16:49:24 -06:00
1 parent a695996091
commit 530e2e302b
5 files changed
+586 -25

No files matched your search

@@ -0,0 +1,128 @@
/*
* Copyright (c) 2026 James Maes
* SPDX-License-Identifier: MIT
*/
using System.Globalization;
namespace Qqq.PrintAgent.Core.Authentication;
/// <summary>
/// Load balancer affinity. A QQQ server may run as several nodes behind a load balancer that names the node it chose in a
/// cookie and expects that cookie back, so one client's requests keep reaching one node. This client attaches its cookies by
/// hand and used to drop everything but the session, so its requests and its realtime socket could land on different nodes.
/// It now keeps the load balancer's own cookies, and nothing else: they are held in memory only, for the life of this client
/// object, sent only to the session origin, and replaced or removed whenever the load balancer rewrites them (which is how it
/// moves a client off a node that stopped). They carry routing, never identity, so they are not saved with the session and
/// signing out neither needs nor clears them.
/// </summary>
public sealed partial class QqqSessionClient
{
private const int MaximumAffinityCookies = 8;
private const int MaximumAffinityValueLength = 4096;
private readonly Lock _affinityLock = new();
private readonly SortedDictionary<string, string> _affinity = new(StringComparer.Ordinal);
/// <summary>How many load balancer cookies are held; for the log and tests, never their values.</summary>
internal int AffinityCookieCount
{
get
{
lock(_affinityLock)
{ return _affinity.Count; }
}
}
/// <summary>
/// Keeps or drops load balancer cookies named in a response's Set-Cookie headers. Only the Application Load Balancer
/// family (names beginning AWSALB) is recognized. A cookie is ignored unless it is scoped to this host and the root path,
/// has a plain bounded value and, when marked Secure, arrived over HTTPS; one the server expired is removed. A malformed
/// header is skipped, because a routing hint must never fail a request.
/// </summary>
private void AdoptAffinity(IEnumerable<string> setCookieHeaders)
{
DateTimeOffset now = _timeProvider.GetUtcNow();
bool secureOrigin = string.Equals(Origin.Scheme, "https", StringComparison.Ordinal);
foreach(string header in setCookieHeaders)
{
string[] parts = header.Split(';');
int separator = parts[0].IndexOf('=', StringComparison.Ordinal);
if(header.Length > 8192 || separator <= 0)
{ continue; }
string name = parts[0][..separator].Trim();
if(!IsAffinityCookieName(name))
{ continue; }
string value = parts[0][(separator + 1)..].Trim();
bool expired = value.Length == 0;
bool usable = value.Length <= MaximumAffinityValueLength && !value.Any(character => character is <= ' ' or '"' or ',' or '\\' or >= (char)0x7f);
foreach(string attribute in parts.Skip(1))
{
string[] pair = attribute.Split('=', 2);
string key = pair[0].Trim();
string detail = pair.Length == 2 ? pair[1].Trim() : "";
if(key.Equals("Max-Age", StringComparison.OrdinalIgnoreCase))
{ expired |= long.TryParse(detail, NumberStyles.AllowLeadingSign, CultureInfo.InvariantCulture, out long seconds) && seconds <= 0; }
else if(key.Equals("Expires", StringComparison.OrdinalIgnoreCase))
{ expired |= DateTimeOffset.TryParse(detail, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out DateTimeOffset at) && at <= now; }
else if(key.Equals("Domain", StringComparison.OrdinalIgnoreCase))
{ usable &= string.Equals(detail.TrimStart('.'), Origin.Host, StringComparison.OrdinalIgnoreCase); }
else if(key.Equals("Path", StringComparison.OrdinalIgnoreCase))
{ usable &= string.Equals(detail, "/", StringComparison.Ordinal); }
else if(key.Equals("Secure", StringComparison.OrdinalIgnoreCase))
{ usable &= secureOrigin; }
}
lock(_affinityLock)
{
if(expired)
{ _affinity.Remove(name); }
else if(usable && (_affinity.ContainsKey(name) || _affinity.Count < MaximumAffinityCookies))
{ _affinity[name] = value; }
}
}
}
/// <summary>The Cookie header for a request to the session origin: the session cookies the caller passed, when any, followed by the load balancer cookies; null when there is neither.</summary>
private string? WithAffinity(string? sessionCookie)
{
string affinity;
lock(_affinityLock)
{
if(_affinity.Count == 0)
{ return sessionCookie; }
affinity = string.Join("; ", _affinity.Select(pair => pair.Key + "=" + pair.Value));
}
return string.IsNullOrEmpty(sessionCookie) ? affinity : sessionCookie + "; " + affinity;
}
/// <summary>
/// Reads the load balancer cookies from a WebSocket upgrade answer and releases the collected headers. It runs whether the
/// upgrade was accepted or refused: a node that is stopping answers 502 or 503, and that answer carries the load
/// balancer's new choice of node for the retry.
/// </summary>
private void AdoptSocketAffinity(System.Net.WebSockets.ClientWebSocket socket)
{
IReadOnlyDictionary<string, IEnumerable<string>>? headers = socket.HttpResponseHeaders;
if(headers is null)
{ return; }
foreach(KeyValuePair<string, IEnumerable<string>> header in headers)
{
if(header.Key.Equals("Set-Cookie", StringComparison.OrdinalIgnoreCase))
{ AdoptAffinity(header.Value); }
}
socket.HttpResponseHeaders = null;
}
/// <summary>True for the Application Load Balancer's cookie names: AWSALB, AWSALBCORS, AWSALBTG, AWSALBTGCORS and the AWSALBAPP series.</summary>
private static bool IsAffinityCookieName(string name) => name.Length <= 64 && name.StartsWith("AWSALB", StringComparison.Ordinal)
&& name.All(character => char.IsAsciiLetterOrDigit(character) || character is '-' or '_');
}
@@ -39,7 +39,9 @@ public sealed partial class QqqSessionClient
var uri = new UriBuilder(new Uri(Origin, PrintingPath + "device/realtime")) { Scheme = string.Equals(Origin.Scheme, "https", StringComparison.Ordinal) ? "wss" : "ws" };
_diagnostics.Info($"Opening the realtime socket to {QqqSessionDiagnostics.Describe(uri.Uri)} with subprotocol qqq-printing.v1 (session cookie and device proof attached; heartbeat every {heartbeatInterval.TotalSeconds:0} s, timeout {heartbeatTimeout.TotalSeconds:0} s).");
long started = System.Diagnostics.Stopwatch.GetTimestamp();
await socket.ConnectAsync(uri.Uri, invoker, cancellationToken).ConfigureAwait(false);
try
{ await socket.ConnectAsync(uri.Uri, invoker, cancellationToken).ConfigureAwait(false); }
finally { AdoptSocketAffinity(socket); }
_diagnostics.Info($"Realtime socket upgraded in {Elapsed(started)}; server selected subprotocol {socket.SubProtocol ?? "(none)"}.");
await _gate.WaitAsync(cancellationToken).ConfigureAwait(false);
try
@@ -77,7 +79,7 @@ public sealed partial class QqqSessionClient
/// <summary>
/// The HTTP invoker for the WebSocket upgrade: no redirects, no cookie jar (the cookie is attached by hand) and the same
/// The HTTP invoker for the WebSocket upgrade: no redirects, no cookie jar (the cookies are attached by hand) and the same
/// certificate policy as the HTTP client, so a rejected or explicitly allowed certificate is logged the same way. See the
/// MA0039 note on the constructor.
/// </summary>
@@ -99,12 +101,16 @@ public sealed partial class QqqSessionClient
/// <summary>Sets the subprotocol, session cookie, origin, client channel, device proof and keep-alive on a socket before it connects; the caller holds the gate.</summary>
/// <summary>
/// Sets the subprotocol, cookies, origin, client channel, device proof and keep-alive on a socket before it connects; the
/// caller holds the gate. The upgrade carries the load balancer's cookies beside the session, so the socket opens on the
/// node this client's HTTP requests already reach, and that node then serves its claims and downloads too.
/// </summary>
private void ConfigureSocket(ClientWebSocket socket, QqqDeviceCredential device, TimeSpan heartbeatInterval, TimeSpan heartbeatTimeout)
{
socket.Options.AddSubProtocol("qqq-printing.v1");
socket.Options.CollectHttpResponseDetails = true;
socket.Options.SetRequestHeader("Cookie", CookieHeader());
socket.Options.SetRequestHeader("Cookie", WithAffinity(CookieHeader()));
socket.Options.SetRequestHeader("Origin", Origin.GetLeftPart(UriPartial.Authority));
socket.Options.SetRequestHeader("X-QQQ-Client", ClientChannel);
socket.Options.SetRequestHeader("X-QQQ-Device-Id", device.DeviceId);
@@ -40,8 +40,9 @@ public sealed partial class QqqSessionClient : IQqqSessionClient
/// <summary>
/// Creates a client bound to one server origin. The origin must be an absolute https URL with no credentials, path,
/// query or fragment (plain http only when explicitly allowed and loopback). The handler never follows redirects or
/// manages cookies, because redirects are refused and the session cookie is attached by hand to each request, and the
/// HTTP client has no overall timeout because every request sets its own bound.
/// manages cookies, because redirects are refused and the cookies this client sends (the session, and a load balancer's
/// affinity cookies) are attached by hand to each request, and the HTTP client has no overall timeout because every
/// request sets its own bound.
/// </summary>
public QqqSessionClient(Uri origin, HttpMessageHandler? handler = null, TimeProvider? timeProvider = null, bool allowLoopbackHttp = false, ApplicationTrafficMeter? traffic = null,
QqqSessionDiagnostics? diagnostics = null, bool allowUntrustedCertificate = false)
@@ -496,7 +497,7 @@ public sealed partial class QqqSessionClient : IQqqSessionClient
}
string route = QqqSessionDiagnostics.Describe(uri);
long started = System.Diagnostics.Stopwatch.GetTimestamp();
_diagnostics.Debug(() => $"HTTP {method} {route}: session cookie {(cookie is not null ? "yes" : "no")}, device proof {(device is not null ? "yes" : "no")}, basic credential {(basic is not null ? "yes" : "no")}, body {(content is null ? "none" : content.Headers.ContentType?.MediaType ?? "present")}, limit {(boundedByCaller ? "caller" : "20 s")}, response bound {maximumBytes} bytes.");
_diagnostics.Debug(() => $"HTTP {method} {route}: session cookie {(cookie is not null ? "yes" : "no")}, load balancer cookies {(providerDiscovery ? 0 : AffinityCookieCount)}, device proof {(device is not null ? "yes" : "no")}, basic credential {(basic is not null ? "yes" : "no")}, body {(content is null ? "none" : content.Headers.ContentType?.MediaType ?? "present")}, limit {(boundedByCaller ? "caller" : "20 s")}, response bound {maximumBytes} bytes.");
using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
if(!boundedByCaller)
{ timeout.CancelAfter(TimeSpan.FromSeconds(20)); }
@@ -514,6 +515,9 @@ public sealed partial class QqqSessionClient : IQqqSessionClient
_diagnostics.Debug(() => $"HTTP {method} {route} -> {(int)response.StatusCode} {response.ReasonPhrase} in {Elapsed(started)}; content type {response.Content.Headers.ContentType?.MediaType ?? "(none)"}, length {response.Content.Headers.ContentLength?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "unknown"}, Set-Cookie {(response.Headers.Contains("Set-Cookie") ? "present" : "absent")}.");
if(!providerDiscovery && _traffic is not null)
{ response.Content = new TrafficHttpContent(response.Content, _traffic.RecordDownload); }
// Every answer from the session origin may rewrite the load balancer's cookies, including an error from a node that is stopping.
if(!providerDiscovery && response.Headers.TryGetValues("Set-Cookie", out IEnumerable<string>? setCookies))
{ AdoptAffinity(setCookies); }
if((int)response.StatusCode is >= 300 and < 400)
{
_diagnostics.Info($"HTTP {method} {route} answered {(int)response.StatusCode} (redirect); redirects are refused, so the request failed. Check the server address: it must be the exact HTTPS origin QQQ serves from.");
@@ -537,7 +541,11 @@ public sealed partial class QqqSessionClient : IQqqSessionClient
/// <summary>Builds the outgoing request: no-store caching, this agent's channel header, and only the cookie, origin, Basic credential and device proof the caller passed.</summary>
/// <summary>
/// Builds the outgoing request: no-store caching, this agent's channel header, and only the session cookie, origin, Basic
/// credential and device proof the caller passed. A request to the session origin also carries the load balancer's
/// cookies, with or without a session, so it reaches the node the load balancer last chose; the provider never sees them.
/// </summary>
private HttpRequestMessage BuildRequest(Uri uri, HttpMethod method, HttpContent? content, string? cookie, QqqDeviceCredential? device, string? basic, bool providerDiscovery)
{
var request = new HttpRequestMessage(method, uri)
@@ -547,13 +555,12 @@ public sealed partial class QqqSessionClient : IQqqSessionClient
request.Headers.CacheControl = new CacheControlHeaderValue { NoStore = true };
if(!providerDiscovery)
{ request.Headers.Add("X-QQQ-Client", ClientChannel); }
if(cookie is not null)
string? cookies = providerDiscovery ? cookie : WithAffinity(cookie);
if(cookies is not null)
{ request.Headers.Add("Cookie", cookies); }
if(cookie is not null && method != HttpMethod.Get && method != HttpMethod.Head && method != HttpMethod.Options)
{
request.Headers.Add("Cookie", cookie);
if(method != HttpMethod.Get && method != HttpMethod.Head && method != HttpMethod.Options)
{
request.Headers.Add("Origin", Origin.GetLeftPart(UriPartial.Authority));
}
request.Headers.Add("Origin", Origin.GetLeftPart(UriPartial.Authority));
}
if(basic is not null)
{ request.Headers.Authorization = new AuthenticationHeaderValue("Basic", basic); }
@@ -647,7 +654,9 @@ public sealed partial class QqqSessionClient : IQqqSessionClient
/// <summary>
/// Extracts the session token from Set-Cookie headers, strictly. Headers are size-bounded, the cookie must be for this
/// host with path "/", and repeated session cookies must agree. An expired, empty or deleted session cookie means the
/// server ended the session, so the local session is cleared and sign-in is required.
/// server ended the session, so the local session is cleared and sign-in is required. Cookies under any other name are
/// not the session and are not examined: a load balancer in front of the server adds its own to every answer, and one
/// this parser dislikes must not turn a good answer into an invalid one.
/// </summary>
private string? ReadSessionCookie(HttpResponseMessage response)
{
@@ -658,12 +667,14 @@ public sealed partial class QqqSessionClient : IQqqSessionClient
{
if(header.Length > 8192)
{ throw InvalidResponse(); }
string name = header.Split('=', 2)[0].Trim();
if(name is not "sessionUUID" and not "sessionId")
{ continue; }
var parsed = new CookieContainer();
try
{ parsed.SetCookies(Origin, header); }
catch(CookieException) { throw InvalidResponse(); }
string name = header.Split('=', 2)[0].Trim();
if((name is "sessionUUID" or "sessionId") && parsed.GetAllCookies().Count == 0)
if(parsed.GetAllCookies().Count == 0)
{
_diagnostics.Info($"The server deleted the {name} cookie; the session has ended and sign-in is required.");
ClearSession();
@@ -708,7 +719,11 @@ public sealed partial class QqqSessionClient : IQqqSessionClient
/// <summary>Forgets the token, cookies, device and bound identity and advances the generation, so every request already in flight is refused when it returns.</summary>
/// <summary>
/// Forgets the token, session cookies, device and bound identity and advances the generation, so every request already in
/// flight is refused when it returns. The load balancer's cookies are kept: they say which node to reach, not who is
/// asking, and the sign-in that follows should reach the same node.
/// </summary>
private void ClearSession()
{
_sessionToken = null;
@@ -0,0 +1,385 @@
/*
* Copyright (c) 2026 James Maes
* SPDX-License-Identifier: MIT
*/
using System.Net;
using System.Net.WebSockets;
using System.Text;
using Qqq.PrintAgent.Core.Authentication;
using Qqq.PrintAgent.Testing;
namespace Qqq.PrintAgent.Core.Tests;
/// <summary>
/// A QQQ server that runs as several nodes sits behind a load balancer which names the node it chose in a cookie. These tests
/// fix what the session client does with that cookie: it keeps the load balancer's cookies and nothing else, sends them on
/// every later request and on the realtime upgrade so claims reach the node that holds the socket, and follows the load
/// balancer when it rewrites them. They also fix what a refused request means when a node stops:
/// only a 401 ends the session, and a 5xx or a dropped connection leaves it signed in for the next attempt.
/// </summary>
public sealed class LoadBalancerAffinityTests
{
private static readonly Uri Origin = new("https://printing.example.invalid");
private const string Discovery = """{"apiVersion":"1.9.0","features":["version","metaData","processes","processCancel","extensions","widgets","nativeSessionLogin"],"clientChannels":["api"],"extensions":[{"name":"printing","version":"2.4.0","pathPrefix":"/qbit/printing"}]}""";
private const string Table = """{"name":"local","type":"TABLE_BASED"}""";
private const string Identity = """{"principalId":"user-1","tenantId":"tenant-1","deviceId":"device-1"}""";
private const string Session = "sessionUUID=session-secret; Path=/; Secure; HttpOnly";
/// <summary>The two cookies an Application Load Balancer with duration stickiness adds to every answer, in its own format.</summary>
private static string[] Balancer(string node) =>
[
$"AWSALB={node}+aB3/xYz==; Expires=Tue, 17 Nov 2099 18:00:00 GMT; Path=/",
$"AWSALBCORS={node}+aB3/xYz==; Expires=Tue, 17 Nov 2099 18:00:00 GMT; Path=/; SameSite=None; Secure",
];
/// <summary>
/// The load balancer's cookies from the first public answer travel on every later request, with or without a session, and
/// never displace or disturb the session cookie. When a later answer rewrites one (the load balancer moved this client to
/// another node), the next request carries the new value. The sign-out request carries them too, so it reaches that node.
/// </summary>
[Fact]
public async Task CookiesFromTheLoadBalancerTravelOnEveryLaterRequestBesideTheSession()
{
using var handler = new Handler();
handler.Add(Discovery, cookies: Balancer("node-one"));
handler.Add(Table);
handler.Add("{}", cookies: [Session, .. Balancer("node-one")]);
handler.Add(Identity);
handler.Add("""{"job":null}""", cookies: Balancer("node-two"));
handler.Add("""{"job":null}""");
handler.Add("{}");
using var client = new QqqSessionClient(Origin, handler);
await client.SignInWithPasswordAsync("user", "password");
Assert.Null(handler.Requests[0].Cookie);
Assert.Equal("AWSALB=node-one+aB3/xYz==; AWSALBCORS=node-one+aB3/xYz==", handler.Requests[1].Cookie);
Assert.Equal("AWSALB=node-one+aB3/xYz==; AWSALBCORS=node-one+aB3/xYz==", handler.Requests[2].Cookie);
Assert.Null(handler.Requests[1].Origin);
Assert.True(client.IsAuthenticated);
Assert.Equal("session-secret", client.ExportCredential().SessionToken);
await client.BindDeviceAsync(new QqqDeviceCredential("device-1", "proof"));
using(HttpResponseMessage first = await client.SendPrintingAsync(HttpMethod.Post, "device/jobs/claim", new StringContent("{}", Encoding.UTF8, "application/json")))
{ Assert.Equal(HttpStatusCode.OK, first.StatusCode); }
string claim = handler.Requests[4].Cookie!;
Assert.Contains("sessionUUID=session-secret", claim, StringComparison.Ordinal);
Assert.Contains("sessionId=session-secret", claim, StringComparison.Ordinal);
Assert.EndsWith("; AWSALB=node-one+aB3/xYz==; AWSALBCORS=node-one+aB3/xYz==", claim, StringComparison.Ordinal);
Assert.Equal(Origin.GetLeftPart(UriPartial.Authority), handler.Requests[4].Origin);
using(HttpResponseMessage second = await client.SendPrintingAsync(HttpMethod.Post, "device/jobs/claim", new StringContent("{}", Encoding.UTF8, "application/json")))
{ Assert.Equal(HttpStatusCode.OK, second.StatusCode); }
Assert.EndsWith("; AWSALB=node-two+aB3/xYz==; AWSALBCORS=node-two+aB3/xYz==", handler.Requests[5].Cookie, StringComparison.Ordinal);
Assert.Equal(2, client.AffinityCookieCount);
await client.SignOutAsync();
Assert.Contains("AWSALB=node-two+aB3/xYz==", handler.Requests[6].Cookie, StringComparison.Ordinal);
Assert.Contains("sessionUUID=session-secret", handler.Requests[6].Cookie, StringComparison.Ordinal);
Assert.False(client.IsAuthenticated);
}
/// <summary>
/// Only the load balancer's own cookies are kept, and only when they are scoped to this host and the root path with a
/// plain value. Any other cookie the server or a proxy sets, and any malformed one, is not stored and is never sent back.
/// </summary>
[Theory]
[InlineData("AWSALB=value; Path=/", true)]
[InlineData("AWSALB=value", true)]
[InlineData("AWSALBTG=value; Expires=Tue, 17 Nov 2099 18:00:00 GMT; Path=/", true)]
[InlineData("AWSALBAPP-0=value; Path=/", true)]
[InlineData("AWSALBCORS=value; Path=/; SameSite=None; Secure", true)]
[InlineData("AWSALB=value; Domain=printing.example.invalid; Path=/", true)]
[InlineData("AWSALB=value; Max-Age=3600; Path=/", true)]
[InlineData("tracking=value; Path=/", false)]
[InlineData("JSESSIONID=value; Path=/", false)]
[InlineData("awsalb=value; Path=/", false)]
[InlineData("AWSALB.other=value; Path=/", false)]
[InlineData("AWSALB=value; Domain=other.example.invalid; Path=/", false)]
[InlineData("AWSALB=value; Path=/elsewhere", false)]
[InlineData("AWSALB=two words; Path=/", false)]
[InlineData("AWSALB=\"quoted\"; Path=/", false)]
[InlineData("AWSALB=; Path=/", false)]
[InlineData("AWSALB=value; Max-Age=0; Path=/", false)]
[InlineData("AWSALB=value; Expires=Thu, 01 Jan 1970 00:00:00 GMT; Path=/", false)]
[InlineData("AWSALB", false)]
[InlineData("=value", false)]
public async Task OnlyTheLoadBalancersOwnWellFormedCookiesAreKept(string setCookie, bool kept)
{
using var handler = new Handler();
handler.Add(Discovery, cookies: [setCookie]);
using var client = new QqqSessionClient(Origin, handler);
await client.DiscoverAsync();
Assert.Equal(kept ? 1 : 0, client.AffinityCookieCount);
}
/// <summary>A value longer than a load balancer ever sends, an oversized header, and a ninth cookie are all refused, so a hostile server cannot make this client carry an unbounded Cookie header.</summary>
[Fact]
public async Task TheHeldCookiesAreBounded()
{
using var handler = new Handler();
handler.Add(Discovery, cookies: ["AWSALB=" + new string('a', 4097) + "; Path=/", "AWSALBTG=" + new string('a', 9000), .. Enumerable.Range(0, 12).Select(index => $"AWSALBAPP-{index}=value; Path=/")]);
using var client = new QqqSessionClient(Origin, handler);
await client.DiscoverAsync();
Assert.Equal(8, client.AffinityCookieCount);
}
/// <summary>The load balancer replaces a cookie by sending it again and removes one by expiring it; an answer without the cookie leaves it alone.</summary>
[Fact]
public async Task ARewrittenCookieReplacesTheOldOneAndAnExpiredOneIsDropped()
{
using var handler = new Handler();
handler.Add(Discovery, cookies: Balancer("node-one"));
handler.Add(Table, cookies: ["AWSALB=node-two; Path=/"]);
handler.Add("{}", cookies: [Session, "AWSALBCORS=gone; Expires=Thu, 01 Jan 1970 00:00:00 GMT; Path=/; SameSite=None; Secure"]);
handler.Add(Identity);
handler.Add(Identity, cookies: ["AWSALB=; Max-Age=0; Path=/"]);
handler.Add(Identity);
using var client = new QqqSessionClient(Origin, handler);
await client.SignInWithPasswordAsync("user", "password");
Assert.Equal("AWSALB=node-two; AWSALBCORS=node-one+aB3/xYz==", handler.Requests[2].Cookie);
await client.GetSessionIdentityAsync();
Assert.EndsWith("; AWSALB=node-two", handler.Requests[3].Cookie, StringComparison.Ordinal);
Assert.DoesNotContain("AWSALBCORS", handler.Requests[3].Cookie, StringComparison.Ordinal);
await client.GetSessionIdentityAsync();
await client.GetSessionIdentityAsync();
Assert.DoesNotContain("AWSALB", handler.Requests[5].Cookie, StringComparison.Ordinal);
Assert.Contains("sessionUUID=session-secret", handler.Requests[5].Cookie, StringComparison.Ordinal);
Assert.True(client.IsAuthenticated);
}
/// <summary>
/// Cookies under other names never decide whether a session answer is valid. A load balancer or proxy in front of the
/// server may add one this client would not accept as a session cookie (here, scoped to another domain); sign-in still
/// succeeds, the foreign cookie is not kept, and the session cookie is still checked as strictly as before.
/// </summary>
[Fact]
public async Task CookiesUnderOtherNamesNeverInvalidateASessionAnswer()
{
using var handler = new Handler();
handler.Add(Discovery);
handler.Add(Table);
handler.Add("{}", cookies: ["edge=abc; Domain=other.example.invalid; Path=/", Session, .. Balancer("node-one")]);
handler.Add(Identity);
using var client = new QqqSessionClient(Origin, handler);
await client.SignInWithPasswordAsync("user", "password");
Assert.True(client.IsAuthenticated);
await client.GetSessionIdentityAsync();
Assert.DoesNotContain("edge=", handler.Requests[3].Cookie, StringComparison.Ordinal);
using var strict = new Handler();
strict.Add(Discovery);
strict.Add(Table);
strict.Add("{}", cookies: ["sessionUUID=session-secret; Domain=other.example.invalid; Path=/", .. Balancer("node-one")]);
using var refused = new QqqSessionClient(Origin, strict);
await Assert.ThrowsAsync<QqqSessionException>(() => refused.SignInWithPasswordAsync("user", "password"));
Assert.False(refused.IsAuthenticated);
}
/// <summary>
/// A node that is stopping answers 5xx, or the load balancer answers 502, 503 or 504 for it. That is not a verdict on the
/// session: the answer is handed back, the session stays signed in, and the load balancer's new choice of node in that
/// answer is adopted for the retry. Only a 401 ends the session.
/// </summary>
[Theory]
[InlineData(HttpStatusCode.BadGateway)]
[InlineData(HttpStatusCode.ServiceUnavailable)]
[InlineData(HttpStatusCode.GatewayTimeout)]
[InlineData(HttpStatusCode.InternalServerError)]
[InlineData(HttpStatusCode.Conflict)]
public async Task AServerErrorKeepsTheSessionAndFollowsTheLoadBalancer(HttpStatusCode status)
{
using Handler handler = SignedIn("node-one");
handler.Add("""{"error":"unavailable"}""", status, Balancer("node-two"));
handler.Add("""{"job":null}""");
handler.Add("{}", HttpStatusCode.Unauthorized);
using var client = new QqqSessionClient(Origin, handler);
await client.SignInWithPasswordAsync("user", "password");
await client.BindDeviceAsync(new QqqDeviceCredential("device-1", "proof"));
using(HttpResponseMessage failed = await client.SendPrintingAsync(HttpMethod.Post, "device/jobs/claim", new StringContent("{}", Encoding.UTF8, "application/json")))
{ Assert.Equal(status, failed.StatusCode); }
Assert.True(client.IsAuthenticated);
using(HttpResponseMessage retried = await client.SendPrintingAsync(HttpMethod.Post, "device/jobs/claim", new StringContent("{}", Encoding.UTF8, "application/json")))
{ Assert.Equal(HttpStatusCode.OK, retried.StatusCode); }
Assert.Contains("AWSALB=node-two+aB3/xYz==", handler.Requests[^1].Cookie, StringComparison.Ordinal);
Assert.Contains("sessionUUID=session-secret", handler.Requests[^1].Cookie, StringComparison.Ordinal);
QqqSessionException ended = await Assert.ThrowsAsync<QqqSessionException>(() => client.SendPrintingAsync(HttpMethod.Post, "device/jobs/claim", new StringContent("{}", Encoding.UTF8, "application/json")));
Assert.Equal(QqqSessionException.FailureKind.AuthenticationRequired, ended.Failure);
Assert.False(client.IsAuthenticated);
}
/// <summary>A connection that drops before any answer (the node died mid-request) surfaces as a transport failure and leaves the session and the held cookies as they were, so the next attempt goes out signed in.</summary>
[Fact]
public async Task ADroppedConnectionKeepsTheSession()
{
using Handler handler = SignedIn("node-one");
handler.AddAsync(_ => throw new HttpRequestException("The connection was reset.", new IOException("reset")));
handler.Add("""{"job":null}""");
using var client = new QqqSessionClient(Origin, handler);
await client.SignInWithPasswordAsync("user", "password");
await client.BindDeviceAsync(new QqqDeviceCredential("device-1", "proof"));
await Assert.ThrowsAsync<HttpRequestException>(() => client.SendPrintingAsync(HttpMethod.Post, "device/jobs/claim", new StringContent("{}", Encoding.UTF8, "application/json")));
Assert.True(client.IsAuthenticated);
using HttpResponseMessage retried = await client.SendPrintingAsync(HttpMethod.Post, "device/jobs/claim", new StringContent("{}", Encoding.UTF8, "application/json"));
Assert.Equal(HttpStatusCode.OK, retried.StatusCode);
Assert.Contains("AWSALB=node-one+aB3/xYz==", handler.Requests[^1].Cookie, StringComparison.Ordinal);
}
/// <summary>
/// The realtime upgrade carries the load balancer's cookie beside the session, so the socket opens on the node the HTTP
/// requests already reach. A cookie on the upgrade answer itself is adopted, so requests made after the socket is open go
/// to the node that holds it. Over loopback HTTP a cookie marked Secure is not kept, as a browser would not keep it.
/// </summary>
[Fact]
public async Task TheRealtimeUpgradeCarriesAndAdoptsTheLoadBalancerCookie()
{
using var peer = new LoopbackWebSocketPeer();
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15));
using var handler = new NativeHandler(["AWSALB=node-one; Path=/", "AWSALBCORS=node-one; Path=/; SameSite=None; Secure"]);
using var session = new QqqSessionClient(peer.Origin, handler, allowLoopbackHttp: true);
await session.SignInWithPasswordAsync("synthetic-user", "synthetic-password", timeout.Token);
await session.BindDeviceAsync(new QqqDeviceCredential("device", "synthetic-proof"), cancellationToken: timeout.Token);
Assert.Equal(1, session.AffinityCookieCount);
peer.AnswerHeaders.Add("Set-Cookie: AWSALB=node-two; Path=/");
Task<WebSocket> accepted = peer.AcceptAsync(timeout.Token);
using WebSocket socket = await session.ConnectPrintingAsync(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10), timeout.Token);
using WebSocket server = await accepted;
Assert.Contains("sessionUUID=synthetic-session", peer.Headers["Cookie"], StringComparison.Ordinal);
Assert.EndsWith("; AWSALB=node-one", peer.Headers["Cookie"], StringComparison.Ordinal);
await session.GetSessionIdentityAsync(timeout.Token);
Assert.EndsWith("; AWSALB=node-two", handler.Cookies[^1], StringComparison.Ordinal);
}
/// <summary>
/// A refused upgrade is read for what it is. A 5xx (the node is stopping, or the load balancer has no healthy node yet) is
/// a transport failure the runtime retries with backoff: the session stays signed in and the load balancer's cookie on the
/// refusal is adopted, so the retry can land on another node. A 401 is the server withdrawing authorization.
/// </summary>
[Theory]
[InlineData("503 Service Unavailable", false)]
[InlineData("502 Bad Gateway", false)]
[InlineData("401 Unauthorized", true)]
public async Task ARefusedUpgradeIsRetryableUnlessAuthorizationWasWithdrawn(string status, bool authorization)
{
using var peer = new LoopbackWebSocketPeer();
using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(15));
using var handler = new NativeHandler(["AWSALB=node-one; Path=/"]);
using var session = new QqqSessionClient(peer.Origin, handler, allowLoopbackHttp: true);
await session.SignInWithPasswordAsync("synthetic-user", "synthetic-password", timeout.Token);
await session.BindDeviceAsync(new QqqDeviceCredential("device", "synthetic-proof"), cancellationToken: timeout.Token);
peer.AnswerHeaders.Add("Set-Cookie: AWSALB=node-two; Path=/");
Task refused = peer.RefuseAsync(status, timeout.Token);
Exception failure = await Assert.ThrowsAnyAsync<Exception>(async () =>
{
using WebSocket socket = await session.ConnectPrintingAsync(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(10), timeout.Token);
});
await refused;
if(authorization)
{ Assert.Equal(QqqSessionException.FailureKind.AuthenticationRequired, Assert.IsType<QqqSessionException>(failure).Failure); }
else
{ Assert.IsType<WebSocketException>(failure); }
Assert.True(session.IsAuthenticated);
Assert.EndsWith("; AWSALB=node-one", peer.Headers["Cookie"], StringComparison.Ordinal);
await session.GetSessionIdentityAsync(timeout.Token);
Assert.EndsWith("; AWSALB=node-two", handler.Cookies[^1], StringComparison.Ordinal);
}
/// <summary>A fake server scripted through discovery, password sign-in and the identity read that binding a device makes, with the load balancer naming a node on the first answer.</summary>
private static Handler SignedIn(string node)
{
var handler = new Handler();
handler.Add(Discovery, cookies: Balancer(node));
handler.Add(Table);
handler.Add("{}", cookies: [Session]);
handler.Add(Identity);
return handler;
}
/// <summary>What the fake server saw for one request: its URL and the Cookie and Origin headers.</summary>
private sealed record Request(Uri Uri, string? Cookie, string? Origin);
/// <summary>Scripted fake server: it records every request and answers them in the order replies were queued.</summary>
private sealed class Handler : HttpMessageHandler
{
private readonly Queue<Func<CancellationToken, Task<HttpResponseMessage>>> _responses = new();
internal List<Request> Requests { get; } = [];
/// <summary>Queues a JSON reply with the given status and Set-Cookie headers.</summary>
internal void Add(string body, HttpStatusCode status = HttpStatusCode.OK, IEnumerable<string>? cookies = null) => AddAsync(_ =>
{
var response = new HttpResponseMessage(status) { Content = new StringContent(body, Encoding.UTF8, "application/json") };
if(cookies is not null)
{ response.Headers.TryAddWithoutValidation("Set-Cookie", cookies); }
return Task.FromResult(response);
});
/// <summary>Queues a reply built on demand, for one that fails instead of answering.</summary>
internal void AddAsync(Func<CancellationToken, Task<HttpResponseMessage>> response) => _responses.Enqueue(response);
/// <summary>Records the request's cookie and origin, then returns the next queued reply.</summary>
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
string? Header(string name) => request.Headers.TryGetValues(name, out IEnumerable<string>? values) ? string.Join("; ", values) : null;
Requests.Add(new Request(request.RequestUri!, Header("Cookie"), Header("Origin")));
return _responses.Dequeue()(cancellationToken);
}
}
/// <summary>Fake native QQQ server for the realtime tests: discovery advertising realtime printing, table-based sign-in and the session identity, each answer carrying the given load balancer cookies; it records the Cookie header of every request.</summary>
private sealed class NativeHandler(string[] balancerCookies) : HttpMessageHandler
{
internal List<string> Cookies { get; } = [];
/// <summary>Serves the scripted routes; any other route is a test failure.</summary>
protected override Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
{
Cookies.Add(request.Headers.TryGetValues("Cookie", out IEnumerable<string>? values) ? string.Join("; ", values) : "");
string body = request.RequestUri!.AbsolutePath switch
{
"/qqq/v1/version" => Discovery,
"/qqq/v1/metaData/authentication" => Table,
"/qqq/v1/manageSession" => """{"uuid":"synthetic-session"}""",
"/qqq/v1/qbit/printing/session" => """{"principalId":"user","tenantId":"tenant","deviceId":"device"}""",
_ => throw new InvalidOperationException("Unexpected HTTP route."),
};
var response = new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(body) };
if(Cookies.Count == 1)
{ response.Headers.TryAddWithoutValidation("Set-Cookie", balancerCookies); }
return Task.FromResult(response);
}
}
}
@@ -21,6 +21,9 @@ internal sealed class LoopbackWebSocketPeer : IDisposable
internal Uri Origin { get; }
internal Dictionary<string, string> Headers { get; } = new(StringComparer.OrdinalIgnoreCase);
/// <summary>Extra header lines for the next answer, such as the Set-Cookie a load balancer adds; empty unless a test fills it.</summary>
internal List<string> AnswerHeaders { get; } = [];
/// <summary>Starts listening on an ephemeral loopback port, so each test gets its own address and nothing else on the machine can connect.</summary>
@@ -37,6 +40,31 @@ internal sealed class LoopbackWebSocketPeer : IDisposable
/// credentials were presented), answers the RFC 6455 handshake and returns the server end of the socket.
/// </summary>
internal async Task<WebSocket> AcceptAsync(CancellationToken token)
{
NetworkStream stream = await ReadUpgradeAsync(token);
// RFC6455 requires SHA-1 for the public handshake challenge; this is not an authentication digest.
#pragma warning disable CA5350
string accept = Convert.ToBase64String(SHA1.HashData(Encoding.ASCII.GetBytes(Headers["Sec-WebSocket-Key"] + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11")));
#pragma warning restore CA5350
byte[] response = Encoding.ASCII.GetBytes($"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\nSec-WebSocket-Protocol: qqq-printing.v1\r\n{ExtraHeaders()}\r\n");
await stream.WriteAsync(response, token);
return WebSocket.CreateFromStream(stream, new WebSocketCreationOptions { IsServer = true, SubProtocol = "qqq-printing.v1" });
}
/// <summary>Accepts one connection, records the upgrade request's headers and refuses the upgrade with the given status line, as a server or a load balancer in front of it does when it cannot serve the socket.</summary>
internal async Task RefuseAsync(string status, CancellationToken token)
{
NetworkStream stream = await ReadUpgradeAsync(token);
await stream.WriteAsync(Encoding.ASCII.GetBytes($"HTTP/1.1 {status}\r\nContent-Length: 0\r\nConnection: close\r\n{ExtraHeaders()}\r\n"), token);
await stream.FlushAsync(token);
}
/// <summary>Accepts one connection, requires the realtime path and records the upgrade request's headers.</summary>
private async Task<NetworkStream> ReadUpgradeAsync(CancellationToken token)
{
TcpClient client = await _listener.AcceptTcpClientAsync(token);
_clients.Add(client);
@@ -51,17 +79,16 @@ internal sealed class LoopbackWebSocketPeer : IDisposable
string[] pair = line.Split(':', 2);
Headers[pair[0]] = pair[1].Trim();
}
// RFC6455 requires SHA-1 for the public handshake challenge; this is not an authentication digest.
#pragma warning disable CA5350
string accept = Convert.ToBase64String(SHA1.HashData(Encoding.ASCII.GetBytes(Headers["Sec-WebSocket-Key"] + "258EAFA5-E914-47DA-95CA-C5AB0DC85B11")));
#pragma warning restore CA5350
byte[] response = Encoding.ASCII.GetBytes($"HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: {accept}\r\nSec-WebSocket-Protocol: qqq-printing.v1\r\n\r\n");
await stream.WriteAsync(response, token);
return WebSocket.CreateFromStream(stream, new WebSocketCreationOptions { IsServer = true, SubProtocol = "qqq-printing.v1" });
return stream;
}
/// <summary>The extra answer header lines, each terminated, or nothing.</summary>
private string ExtraHeaders() => string.Concat(AnswerHeaders.Select(header => header + "\r\n"));
/// <summary>Sends one realtime message in the protocol envelope (type, a fresh message id and the payload) as a single text frame.</summary>
internal static Task SendAsync(WebSocket socket, string type, object payload, CancellationToken token)
=> socket.SendAsync(JsonSerializer.SerializeToUtf8Bytes(new { type, messageId = Guid.NewGuid().ToString("N"), payload }, PrintApiClient.JsonOptions), WebSocketMessageType.Text, true, token);