From 2570e1775059328ecf8822d4dea597a2f77becbe Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 10 Nov 2026 18:44:06 -0600 Subject: [PATCH 1/3] fix(core): a reader overtaken by the caller's cancellation does not fail the run RunAsync returns when its caller cancels and throws when the channel ended by itself. Its shutdown aborts the socket, and a loop that has checked the token and is about to use the socket at that moment fails on the aborted socket instead of seeing the cancellation. The run then threw that failure although nothing had failed, so a deliberate stop could be reported as a dropped connection. A full verification run hit it once in ErrorFailsTheWaitingSendWithTheMatchingStatus. A loop that had not failed when the shutdown began, and then fails the way an aborted socket fails (disposed, or a socket error), is now the shutdown itself. Any other failure is still reported whenever it arrives, and a loop that had already failed before the shutdown is reported whatever it failed with. The scripted socket gains a hook at the start of a receive, and a new test holds the reader there until the socket is aborted. It fails on the old code every time. Co-Authored-By: Claude Fable 5.2 Signed-off-by: James Maes --- .../Printing/PrintingRealtimeChannel.cs | 31 +++++++++++++++++-- .../RealtimeChannelEndingTests.cs | 31 +++++++++++++++++++ .../ScriptedWebSocket.cs | 6 ++++ 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs b/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs index ee8382c..9489a9e 100644 --- a/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs +++ b/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs @@ -5,6 +5,7 @@ using System.Collections.Concurrent; using System.Net.WebSockets; +using System.Runtime.ExceptionServices; using System.Text.Json; using System.Threading.Channels; using Qqq.PrintAgent.Core.Transport; @@ -56,6 +57,12 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l /// cancellation while the caller's token is not cancelled ends the method as that cancellation, because it was not /// the caller who asked to stop. /// + /// + /// The shutdown aborts the socket, and a loop that is about to use the socket at that moment fails on it instead of + /// seeing the cancellation. That failure is the shutdown itself and is not reported: a loop that had not failed when + /// the shutdown began, and then fails the way an aborted socket fails, did not end the channel. Any other failure is + /// reported whenever it arrives, even beside a cancellation. + /// /// public async Task RunAsync(Action received, CancellationToken token) { @@ -69,6 +76,8 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l // was disposed a moment later still reports its failure // //////////////////////////////////////////////////////////////////////// bool disposed = Volatile.Read(ref _disposed) != 0; + bool readerFailedFirst = reader.IsFaulted; + bool writerFailedFirst = writer.IsFaulted; Interlocked.Exchange(ref _closed, 1); _outgoing.Writer.TryComplete(); await lifetime.CancelAsync().ConfigureAwait(false); @@ -87,8 +96,26 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l // caller with neither loop having failed. anything else is rethrown: // // a failure, the reader's first, or a cancellation nobody asked for // ///////////////////////////////////////////////////////////////////////// - if(!disposed && !(both.IsCanceled && token.IsCancellationRequested)) - { await both.ConfigureAwait(false); } + if(disposed) + { return; } + Exception? failure = FailureOf(reader, readerFailedFirst) ?? FailureOf(writer, writerFailedFirst); + if(failure is not null) + { ExceptionDispatchInfo.Throw(failure); } + if(!token.IsCancellationRequested) + { await (reader.IsCanceled ? reader : writer.IsCanceled ? writer : both).ConfigureAwait(false); } + } + + + + /// + /// What a loop failed with, or null when it did not fail or its failure is only the shutdown: a loop that had not + /// failed when the shutdown began and then failed the way an aborted socket fails (disposed, or a socket error) was + /// stopped by the abort, not by anything the server or the network did. + /// + private static Exception? FailureOf(Task loop, bool failedBeforeShutdown) + { + Exception? failure = loop.Exception?.InnerException; + return failure is null || (!failedBeforeShutdown && failure is ObjectDisposedException or WebSocketException) ? null : failure; } diff --git a/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs b/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs index 6006594..6de73e1 100644 --- a/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs +++ b/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs @@ -181,6 +181,37 @@ public sealed class RealtimeChannelEndingTests + /// + /// The caller's cancellation can overtake the reader: the reader has checked the token and is about to ask the socket + /// for a frame when the caller cancels and the shutdown aborts the socket, so the reader fails on the aborted socket + /// instead of seeing the cancellation. Nothing failed, so the run still returns. The hook holds the reader at exactly + /// that point until the socket has been aborted. + /// + [Fact] + public async Task CallerCancellationThatOvertakesTheReaderStillEndsTheRunNormally() + { + using var stop = new CancellationTokenSource(); + using var socket = new ScriptedWebSocket(); + int receives = 0; + socket.Receiving = () => + { + if(Interlocked.Increment(ref receives) == 2) + { + stop.Cancel(); + Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience)); + } + }; + using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings()); + Task run = channel.RunAsync(_ => { }, stop.Token); + socket.Deliver("PONG", "pong-1", new { serverTime = "2026-01-01T00:00:00Z" }); + await run.WaitAsync(Patience); + Assert.True(run.IsCompletedSuccessfully); + Assert.Equal(2, Volatile.Read(ref receives)); + Assert.Equal(1, socket.Aborts); + } + + + /// A failure is not hidden by a cancellation that arrives with it: when the socket refuses a frame just as the caller cancels, the run still ends with the failure. [Fact] public async Task FailureIsReportedEvenWhenTheCallerCancelsAtTheSameMoment() diff --git a/tests/Qqq.PrintAgent.Core.Tests/ScriptedWebSocket.cs b/tests/Qqq.PrintAgent.Core.Tests/ScriptedWebSocket.cs index a1c52ed..95c4b6e 100644 --- a/tests/Qqq.PrintAgent.Core.Tests/ScriptedWebSocket.cs +++ b/tests/Qqq.PrintAgent.Core.Tests/ScriptedWebSocket.cs @@ -29,6 +29,11 @@ internal sealed class ScriptedWebSocket : WebSocket + /// Called each time the channel asks for a frame, before the socket looks at its own state; a test uses it to act at the moment the reader is about to receive. + internal Action? Receiving { get; set; } + + + /// Called for every frame the channel sends, before it is recorded; a test uses it to stall or fail the send. internal Func? Sending { get; set; } @@ -166,6 +171,7 @@ internal sealed class ScriptedWebSocket : WebSocket /// public override async Task ReceiveAsync(ArraySegment buffer, CancellationToken cancellationToken) { + Receiving?.Invoke(); ThrowIfEnded(); if(!await _incoming.Reader.WaitToReadAsync(cancellationToken)) { throw new WebSocketException(WebSocketError.ConnectionClosedPrematurely); } -- 2.54.0 From 3c47c15a17cec414a0bed7cc444e70b078f56218 Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 10 Nov 2026 19:27:15 -0600 Subject: [PATCH 2/3] fix(core): a socket refusing a call as it ends is the stop, whoever ended it Review round on the channel shutdown race. The platform socket aborts itself as soon as the token of a call in flight is cancelled, before the channel's own shutdown has looked at the loops. A loop that then used the socket failed before the snapshot and was reported as a failure of the run although the caller had asked for the stop. The snapshot now records whether the caller had cancelled, and a loop counts as failed first only when it had not. The failure that is taken to be the stop is narrowed to the two ways an ended socket refuses a call: disposed, or in an invalid state. A connection the server dropped (closed prematurely) and any other error are reported whenever they arrive, also beside a cancellation. Tests pin the rule by kind of failure, and the case where the refusal arrives before the shutdown begins. Co-Authored-By: Claude Fable 5.2 Signed-off-by: James Maes --- .../Printing/PrintingRealtimeChannel.cs | 39 ++++++---- .../RealtimeChannelEndingTests.cs | 74 +++++++++++++++++++ 2 files changed, 98 insertions(+), 15 deletions(-) diff --git a/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs b/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs index 9489a9e..cde812c 100644 --- a/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs +++ b/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs @@ -58,10 +58,12 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l /// the caller who asked to stop. /// /// - /// The shutdown aborts the socket, and a loop that is about to use the socket at that moment fails on it instead of - /// seeing the cancellation. That failure is the shutdown itself and is not reported: a loop that had not failed when - /// the shutdown began, and then fails the way an aborted socket fails, did not end the channel. Any other failure is - /// reported whenever it arrives, even beside a cancellation. + /// A socket that is being ended refuses the next call on it, and a loop that is about to use the socket at that + /// moment fails on it instead of seeing the cancellation. The socket is ended twice over on a deliberate stop: the + /// platform socket aborts itself as soon as the token of a call in flight is cancelled, and this shutdown aborts it + /// again. A failure of that kind (the socket is disposed, or in an invalid state) is the stop itself and is not + /// reported when the caller had cancelled, or when the loop had not failed before this shutdown began. Any other + /// failure is reported whenever it arrives, even beside a cancellation. /// /// public async Task RunAsync(Action received, CancellationToken token) @@ -72,12 +74,16 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l Task ended = await Task.WhenAny(reader, writer).ConfigureAwait(false); //////////////////////////////////////////////////////////////////////// - // read before the teardown below, so a channel that failed first and // - // was disposed a moment later still reports its failure // + // read before the teardown below: whether the channel was disposed, // + // so one that failed first and was disposed a moment later still // + // reports its failure; and which loop had already failed while the // + // caller had not cancelled, because only such a failure can be told // + // from the socket refusing a call as it is being ended // //////////////////////////////////////////////////////////////////////// bool disposed = Volatile.Read(ref _disposed) != 0; - bool readerFailedFirst = reader.IsFaulted; - bool writerFailedFirst = writer.IsFaulted; + bool callerCancelled = token.IsCancellationRequested; + bool readerFailedFirst = reader.IsFaulted && !callerCancelled; + bool writerFailedFirst = writer.IsFaulted && !callerCancelled; Interlocked.Exchange(ref _closed, 1); _outgoing.Writer.TryComplete(); await lifetime.CancelAsync().ConfigureAwait(false); @@ -93,8 +99,9 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l ///////////////////////////////////////////////////////////////////////// // ended on purpose: disposed (whatever the loops met on a socket that // // was disposed under them is the dispose itself), or cancelled by the // - // caller with neither loop having failed. anything else is rethrown: // - // a failure, the reader's first, or a cancellation nobody asked for // + // caller with no failure but the socket refusing a call as it ended. // + // anything else is rethrown: a failure, the reader's first, or a // + // cancellation nobody asked for // ///////////////////////////////////////////////////////////////////////// if(disposed) { return; } @@ -108,14 +115,16 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l /// - /// What a loop failed with, or null when it did not fail or its failure is only the shutdown: a loop that had not - /// failed when the shutdown began and then failed the way an aborted socket fails (disposed, or a socket error) was - /// stopped by the abort, not by anything the server or the network did. + /// What a loop failed with, or null when it did not fail or its failure is only the stop. A socket that has been + /// aborted refuses a call in one of two ways: it is disposed, or it is in an invalid state. A loop that met that, + /// and is not known to have failed before the stop, was stopped by the abort and not by anything the server or the + /// network did. A connection the server dropped fails differently (closed prematurely) and is always reported. /// - private static Exception? FailureOf(Task loop, bool failedBeforeShutdown) + private static Exception? FailureOf(Task loop, bool failedBeforeStop) { Exception? failure = loop.Exception?.InnerException; - return failure is null || (!failedBeforeShutdown && failure is ObjectDisposedException or WebSocketException) ? null : failure; + bool refusedByAnEndedSocket = failure is ObjectDisposedException || (failure is WebSocketException { WebSocketErrorCode: WebSocketError.InvalidState }); + return failure is null || (!failedBeforeStop && refusedByAnEndedSocket) ? null : failure; } diff --git a/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs b/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs index 6de73e1..6aa83b8 100644 --- a/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs +++ b/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs @@ -212,6 +212,80 @@ public sealed class RealtimeChannelEndingTests + /// + /// What a loop meets after the shutdown has aborted the socket decides whether the run fails. The writer is held in + /// its send until the caller has cancelled and the socket has been aborted, and then fails. A socket that refuses the + /// call because it has ended (disposed, or in an invalid state) is the stop itself, so the run returns. Anything + /// else is a failure and is reported although it arrived after the cancellation: an input or output error, and a + /// connection the server dropped, which the socket reports as closed prematurely. + /// + [Theory] + [InlineData("disposed", false)] + [InlineData("invalid-state", false)] + [InlineData("closed-prematurely", true)] + [InlineData("io", true)] + public async Task FailureMetAfterTheSocketWasAbortedIsReportedUnlessItIsTheSocketRefusingTheCall(string kind, bool reported) + { + Exception met = kind switch + { + "disposed" => new ObjectDisposedException("socket"), + "invalid-state" => new WebSocketException(WebSocketError.InvalidState), + "closed-prematurely" => new WebSocketException(WebSocketError.ConnectionClosedPrematurely), + _ => new IOException("Synthetic send failure."), + }; + using var stop = new CancellationTokenSource(); + using var socket = new ScriptedWebSocket(); + socket.Sending = _ => + { + stop.Cancel(); + Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience)); + throw met; + }; + using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings()); + Task run = channel.RunAsync(_ => { }, stop.Token); + await channel.SendAsync("HEARTBEAT", new { }); + if(reported) + { + Assert.Same(met, await Assert.ThrowsAnyAsync(() => run.WaitAsync(Patience))); + } + else + { + await run.WaitAsync(Patience); + Assert.True(run.IsCompletedSuccessfully); + } + } + + + + /// + /// The platform socket aborts itself as soon as the token of a call in flight is cancelled, before this channel's own + /// shutdown begins, and from then on refuses a new call as being in an invalid state. A writer that is about to send + /// at that moment therefore fails before the shutdown has looked at the loops. The reader is held in its callback so + /// the writer is certain to be the loop that ends first. The caller had cancelled, so the refusal is still the stop + /// and the run returns. + /// + [Fact] + public async Task SocketRefusingACallBecauseTheCallerCancelledIsTheStopEvenBeforeTheShutdownBegins() + { + using var stop = new CancellationTokenSource(); + using var socket = new ScriptedWebSocket(); + using var writerFailed = new ManualResetEventSlim(); + socket.Sending = _ => + { + stop.Cancel(); + writerFailed.Set(); + throw new WebSocketException(WebSocketError.InvalidState); + }; + using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings()); + Task run = channel.RunAsync(_ => Assert.True(writerFailed.Wait(Patience)), stop.Token); + socket.Deliver("PONG", "pong-1", new { serverTime = "2026-01-01T00:00:00Z" }); + await channel.SendAsync("HEARTBEAT", new { }); + await run.WaitAsync(Patience); + Assert.True(run.IsCompletedSuccessfully); + } + + + /// A failure is not hidden by a cancellation that arrives with it: when the socket refuses a frame just as the caller cancels, the run still ends with the failure. [Fact] public async Task FailureIsReportedEvenWhenTheCallerCancelsAtTheSameMoment() -- 2.54.0 From 3053200102d1c67b67678596ee4c7742dbd44ba4 Mon Sep 17 00:00:00 2001 From: James Maes Date: Tue, 10 Nov 2026 19:39:22 -0600 Subject: [PATCH 3/3] test(core): make the early-refusal test certain and pin the failed-first rule Second review round on the channel shutdown race. The test for a refusal that arrives before the shutdown begins released the reader as soon as the writer was about to fail, so the reader could end first and the test passed about half the time with the fix removed. The reader is now held until the shutdown aborts the socket, which only follows a loop ending. Nothing pinned the reads of which loop had failed first once the caller's cancellation was taken into account. A new test has no cancellation at all: the callback gives up while the writer is in a send, and the writer's late refusal must not replace the cancellation. A send still waiting for its acknowledgement is no longer given, as its cause, a refusal the run itself took to be the caller's stop. The traffic test on a real socket no longer tolerates a socket failure when its run is stopped; a deliberate stop ends the run normally. Co-Authored-By: Claude Fable 5.2 Signed-off-by: James Maes --- .../Printing/PrintingRealtimeChannel.cs | 8 +++- .../ApplicationTrafficTransportTests.cs | 4 +- .../RealtimeChannelEndingTests.cs | 45 ++++++++++++++++--- 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs b/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs index cde812c..3ee8faa 100644 --- a/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs +++ b/src/Qqq.PrintAgent.Core/Printing/PrintingRealtimeChannel.cs @@ -88,7 +88,13 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l _outgoing.Writer.TryComplete(); await lifetime.CancelAsync().ConfigureAwait(false); socket.Abort(); - Exception? cause = ended.Exception?.InnerException; + + /////////////////////////////////////////////////////////////////////// + // a send still waiting for its acknowledgement is told why the // + // channel closed. a refusal the run takes to be the caller's stop // + // is not a cause, so the waiting send and the run say the same // + /////////////////////////////////////////////////////////////////////// + Exception? cause = FailureOf(ended, !callerCancelled); foreach(TaskCompletionSource pending in _pending.Values) { pending.TrySetException(new RealtimeChannelClosedException("Realtime connection closed before acknowledgement; durable receipts will be replayed.", cause)); diff --git a/tests/Qqq.PrintAgent.Core.Tests/ApplicationTrafficTransportTests.cs b/tests/Qqq.PrintAgent.Core.Tests/ApplicationTrafficTransportTests.cs index 5093642..f3d375c 100644 --- a/tests/Qqq.PrintAgent.Core.Tests/ApplicationTrafficTransportTests.cs +++ b/tests/Qqq.PrintAgent.Core.Tests/ApplicationTrafficTransportTests.cs @@ -119,9 +119,7 @@ public sealed class ApplicationTrafficTransportTests finally { await timeout.CancelAsync(); - try - { await run; } - catch(Exception exception) when(exception is OperationCanceledException or WebSocketException) { } + await run; } } diff --git a/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs b/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs index 6aa83b8..ba6f532 100644 --- a/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs +++ b/tests/Qqq.PrintAgent.Core.Tests/RealtimeChannelEndingTests.cs @@ -260,24 +260,23 @@ public sealed class RealtimeChannelEndingTests /// /// The platform socket aborts itself as soon as the token of a call in flight is cancelled, before this channel's own /// shutdown begins, and from then on refuses a new call as being in an invalid state. A writer that is about to send - /// at that moment therefore fails before the shutdown has looked at the loops. The reader is held in its callback so - /// the writer is certain to be the loop that ends first. The caller had cancelled, so the refusal is still the stop - /// and the run returns. + /// at that moment therefore fails before the shutdown has looked at the loops. The reader is held in its callback + /// until the shutdown aborts the socket, which it does only after a loop has ended, so the writer is the loop that + /// ended first and had already failed when the shutdown looked. The caller had cancelled, so the refusal is still + /// the stop and the run returns. /// [Fact] public async Task SocketRefusingACallBecauseTheCallerCancelledIsTheStopEvenBeforeTheShutdownBegins() { using var stop = new CancellationTokenSource(); using var socket = new ScriptedWebSocket(); - using var writerFailed = new ManualResetEventSlim(); socket.Sending = _ => { stop.Cancel(); - writerFailed.Set(); throw new WebSocketException(WebSocketError.InvalidState); }; using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings()); - Task run = channel.RunAsync(_ => Assert.True(writerFailed.Wait(Patience)), stop.Token); + Task run = channel.RunAsync(_ => Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience)), stop.Token); socket.Deliver("PONG", "pong-1", new { serverTime = "2026-01-01T00:00:00Z" }); await channel.SendAsync("HEARTBEAT", new { }); await run.WaitAsync(Patience); @@ -286,6 +285,40 @@ public sealed class RealtimeChannelEndingTests + /// + /// Without a cancellation by the caller, a loop is taken to have been stopped by the shutdown only when it had not + /// failed before the shutdown began. Nobody cancels here: the callback gives up, as the service's does when a newer + /// connection has replaced this one, while the writer is inside a send. The shutdown then aborts the socket, and the + /// writer's send fails on it. That refusal came after the shutdown began, so it is the shutdown itself, and the run + /// ends as the cancellation the callback raised and not as the socket's refusal. + /// + [Fact] + public async Task RefusalMetAfterTheShutdownBeganIsNotReportedAlthoughNobodyCancelled() + { + using var never = new CancellationTokenSource(); + using var socket = new ScriptedWebSocket(); + using var sending = new ManualResetEventSlim(); + socket.Sending = _ => + { + sending.Set(); + Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience)); + return Task.CompletedTask; + }; + using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings()); + Task run = channel.RunAsync(_ => + { + Assert.True(sending.Wait(Patience)); + throw new OperationCanceledException("Replaced by a newer connection."); + }, never.Token); + await channel.SendAsync("HEARTBEAT", new { }); + socket.Deliver("JOB_AVAILABLE", "wake-1", new { }); + await Assert.ThrowsAnyAsync(() => run.WaitAsync(Patience)); + Assert.True(run.IsCanceled); + Assert.False(never.IsCancellationRequested); + } + + + /// A failure is not hidden by a cancellation that arrives with it: when the socket refuses a frame just as the caller cancels, the run still ends with the failure. [Fact] public async Task FailureIsReportedEvenWhenTheCallerCancelsAtTheSameMoment() -- 2.54.0