fix(core): a stop that overtakes the realtime channel loops does not fail the run #40

Merged
james.maes merged 3 commits from fix/2.0-channel-cancel-race-20261110 into release/2.0 2026-11-11 01:42:56 +00:00
4 changed files with 194 additions and 10 deletions

No files matched your search

@@ -5,6 +5,7 @@
using System.Collections.Concurrent;
using System.Net.WebSockets;
using System.Runtime.ExceptionServices;
using System.Text.Json;
using System.Threading.Channels;
using Qqq.PrintAgent.Core.Transport;
@@ -56,6 +57,14 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l
/// cancellation while the caller's token is not cancelled ends the method as that cancellation, because it was not
/// the caller who asked to stop.
/// </para>
/// <para>
/// A socket that is being ended refuses the next call on it, and a loop that is about to use the socket at that
/// moment fails on it instead of seeing the cancellation. The socket is ended twice over on a deliberate stop: the
/// platform socket aborts itself as soon as the token of a call in flight is cancelled, and this shutdown aborts it
/// again. A failure of that kind (the socket is disposed, or in an invalid state) is the stop itself and is not
/// reported when the caller had cancelled, or when the loop had not failed before this shutdown began. Any other
/// failure is reported whenever it arrives, even beside a cancellation.
/// </para>
/// </summary>
public async Task RunAsync(Action<RealtimeEnvelope> received, CancellationToken token)
{
@@ -65,15 +74,27 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l
Task ended = await Task.WhenAny(reader, writer).ConfigureAwait(false);
////////////////////////////////////////////////////////////////////////
// read before the teardown below, so a channel that failed first and //
// was disposed a moment later still reports its failure //
// read before the teardown below: whether the channel was disposed, //
// so one that failed first and was disposed a moment later still //
// reports its failure; and which loop had already failed while the //
// caller had not cancelled, because only such a failure can be told //
// from the socket refusing a call as it is being ended //
////////////////////////////////////////////////////////////////////////
bool disposed = Volatile.Read(ref _disposed) != 0;
bool callerCancelled = token.IsCancellationRequested;
bool readerFailedFirst = reader.IsFaulted && !callerCancelled;
bool writerFailedFirst = writer.IsFaulted && !callerCancelled;
Interlocked.Exchange(ref _closed, 1);
_outgoing.Writer.TryComplete();
await lifetime.CancelAsync().ConfigureAwait(false);
socket.Abort();
Exception? cause = ended.Exception?.InnerException;
///////////////////////////////////////////////////////////////////////
// a send still waiting for its acknowledgement is told why the //
// channel closed. a refusal the run takes to be the caller's stop //
// is not a cause, so the waiting send and the run say the same //
///////////////////////////////////////////////////////////////////////
Exception? cause = FailureOf(ended, !callerCancelled);
foreach(TaskCompletionSource pending in _pending.Values)
{
pending.TrySetException(new RealtimeChannelClosedException("Realtime connection closed before acknowledgement; durable receipts will be replayed.", cause));
@@ -84,11 +105,32 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l
/////////////////////////////////////////////////////////////////////////
// ended on purpose: disposed (whatever the loops met on a socket that //
// was disposed under them is the dispose itself), or cancelled by the //
// caller with neither loop having failed. anything else is rethrown: //
// a failure, the reader's first, or a cancellation nobody asked for //
// caller with no failure but the socket refusing a call as it ended. //
// anything else is rethrown: a failure, the reader's first, or a //
// cancellation nobody asked for //
/////////////////////////////////////////////////////////////////////////
if(!disposed && !(both.IsCanceled && token.IsCancellationRequested))
{ await both.ConfigureAwait(false); }
if(disposed)
{ return; }
Exception? failure = FailureOf(reader, readerFailedFirst) ?? FailureOf(writer, writerFailedFirst);
if(failure is not null)
{ ExceptionDispatchInfo.Throw(failure); }
if(!token.IsCancellationRequested)
{ await (reader.IsCanceled ? reader : writer.IsCanceled ? writer : both).ConfigureAwait(false); }
}
/// <summary>
/// What a loop failed with, or null when it did not fail or its failure is only the stop. A socket that has been
/// aborted refuses a call in one of two ways: it is disposed, or it is in an invalid state. A loop that met that,
/// and is not known to have failed before the stop, was stopped by the abort and not by anything the server or the
/// network did. A connection the server dropped fails differently (closed prematurely) and is always reported.
/// </summary>
private static Exception? FailureOf(Task loop, bool failedBeforeStop)
{
Exception? failure = loop.Exception?.InnerException;
bool refusedByAnEndedSocket = failure is ObjectDisposedException || (failure is WebSocketException { WebSocketErrorCode: WebSocketError.InvalidState });
return failure is null || (!failedBeforeStop && refusedByAnEndedSocket) ? null : failure;
}
@@ -119,9 +119,7 @@ public sealed class ApplicationTrafficTransportTests
finally
{
await timeout.CancelAsync();
try
{ await run; }
catch(Exception exception) when(exception is OperationCanceledException or WebSocketException) { }
await run;
}
}
@@ -181,6 +181,144 @@ public sealed class RealtimeChannelEndingTests
/// <summary>
/// The caller's cancellation can overtake the reader: the reader has checked the token and is about to ask the socket
/// for a frame when the caller cancels and the shutdown aborts the socket, so the reader fails on the aborted socket
/// instead of seeing the cancellation. Nothing failed, so the run still returns. The hook holds the reader at exactly
/// that point until the socket has been aborted.
/// </summary>
[Fact]
public async Task CallerCancellationThatOvertakesTheReaderStillEndsTheRunNormally()
{
using var stop = new CancellationTokenSource();
using var socket = new ScriptedWebSocket();
int receives = 0;
socket.Receiving = () =>
{
if(Interlocked.Increment(ref receives) == 2)
{
stop.Cancel();
Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience));
}
};
using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings());
Task run = channel.RunAsync(_ => { }, stop.Token);
socket.Deliver("PONG", "pong-1", new { serverTime = "2026-01-01T00:00:00Z" });
await run.WaitAsync(Patience);
Assert.True(run.IsCompletedSuccessfully);
Assert.Equal(2, Volatile.Read(ref receives));
Assert.Equal(1, socket.Aborts);
}
/// <summary>
/// What a loop meets after the shutdown has aborted the socket decides whether the run fails. The writer is held in
/// its send until the caller has cancelled and the socket has been aborted, and then fails. A socket that refuses the
/// call because it has ended (disposed, or in an invalid state) is the stop itself, so the run returns. Anything
/// else is a failure and is reported although it arrived after the cancellation: an input or output error, and a
/// connection the server dropped, which the socket reports as closed prematurely.
/// </summary>
[Theory]
[InlineData("disposed", false)]
[InlineData("invalid-state", false)]
[InlineData("closed-prematurely", true)]
[InlineData("io", true)]
public async Task FailureMetAfterTheSocketWasAbortedIsReportedUnlessItIsTheSocketRefusingTheCall(string kind, bool reported)
{
Exception met = kind switch
{
"disposed" => new ObjectDisposedException("socket"),
"invalid-state" => new WebSocketException(WebSocketError.InvalidState),
"closed-prematurely" => new WebSocketException(WebSocketError.ConnectionClosedPrematurely),
_ => new IOException("Synthetic send failure."),
};
using var stop = new CancellationTokenSource();
using var socket = new ScriptedWebSocket();
socket.Sending = _ =>
{
stop.Cancel();
Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience));
throw met;
};
using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings());
Task run = channel.RunAsync(_ => { }, stop.Token);
await channel.SendAsync("HEARTBEAT", new { });
if(reported)
{
Assert.Same(met, await Assert.ThrowsAnyAsync<Exception>(() => run.WaitAsync(Patience)));
}
else
{
await run.WaitAsync(Patience);
Assert.True(run.IsCompletedSuccessfully);
}
}
/// <summary>
/// The platform socket aborts itself as soon as the token of a call in flight is cancelled, before this channel's own
/// shutdown begins, and from then on refuses a new call as being in an invalid state. A writer that is about to send
/// at that moment therefore fails before the shutdown has looked at the loops. The reader is held in its callback
/// until the shutdown aborts the socket, which it does only after a loop has ended, so the writer is the loop that
/// ended first and had already failed when the shutdown looked. The caller had cancelled, so the refusal is still
/// the stop and the run returns.
/// </summary>
[Fact]
public async Task SocketRefusingACallBecauseTheCallerCancelledIsTheStopEvenBeforeTheShutdownBegins()
{
using var stop = new CancellationTokenSource();
using var socket = new ScriptedWebSocket();
socket.Sending = _ =>
{
stop.Cancel();
throw new WebSocketException(WebSocketError.InvalidState);
};
using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings());
Task run = channel.RunAsync(_ => Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience)), stop.Token);
socket.Deliver("PONG", "pong-1", new { serverTime = "2026-01-01T00:00:00Z" });
await channel.SendAsync("HEARTBEAT", new { });
await run.WaitAsync(Patience);
Assert.True(run.IsCompletedSuccessfully);
}
/// <summary>
/// Without a cancellation by the caller, a loop is taken to have been stopped by the shutdown only when it had not
/// failed before the shutdown began. Nobody cancels here: the callback gives up, as the service's does when a newer
/// connection has replaced this one, while the writer is inside a send. The shutdown then aborts the socket, and the
/// writer's send fails on it. That refusal came after the shutdown began, so it is the shutdown itself, and the run
/// ends as the cancellation the callback raised and not as the socket's refusal.
/// </summary>
[Fact]
public async Task RefusalMetAfterTheShutdownBeganIsNotReportedAlthoughNobodyCancelled()
{
using var never = new CancellationTokenSource();
using var socket = new ScriptedWebSocket();
using var sending = new ManualResetEventSlim();
socket.Sending = _ =>
{
sending.Set();
Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience));
return Task.CompletedTask;
};
using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings());
Task run = channel.RunAsync(_ =>
{
Assert.True(sending.Wait(Patience));
throw new OperationCanceledException("Replaced by a newer connection.");
}, never.Token);
await channel.SendAsync("HEARTBEAT", new { });
socket.Deliver("JOB_AVAILABLE", "wake-1", new { });
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => run.WaitAsync(Patience));
Assert.True(run.IsCanceled);
Assert.False(never.IsCancellationRequested);
}
/// <summary>A failure is not hidden by a cancellation that arrives with it: when the socket refuses a frame just as the caller cancels, the run still ends with the failure.</summary>
[Fact]
public async Task FailureIsReportedEvenWhenTheCallerCancelsAtTheSameMoment()
@@ -29,6 +29,11 @@ internal sealed class ScriptedWebSocket : WebSocket
/// <summary>Called each time the channel asks for a frame, before the socket looks at its own state; a test uses it to act at the moment the reader is about to receive.</summary>
internal Action? Receiving { get; set; }
/// <summary>Called for every frame the channel sends, before it is recorded; a test uses it to stall or fail the send.</summary>
internal Func<CancellationToken, Task>? Sending { get; set; }
@@ -166,6 +171,7 @@ internal sealed class ScriptedWebSocket : WebSocket
/// </summary>
public override async Task<WebSocketReceiveResult> ReceiveAsync(ArraySegment<byte> buffer, CancellationToken cancellationToken)
{
Receiving?.Invoke();
ThrowIfEnded();
if(!await _incoming.Reader.WaitToReadAsync(cancellationToken))
{ throw new WebSocketException(WebSocketError.ConnectionClosedPrematurely); }