fix(core): a stop that overtakes the realtime channel loops does not fail the run #40
No files matched your search
@@ -5,6 +5,7 @@
|
||||
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net.WebSockets;
|
||||
using System.Runtime.ExceptionServices;
|
||||
using System.Text.Json;
|
||||
using System.Threading.Channels;
|
||||
using Qqq.PrintAgent.Core.Transport;
|
||||
@@ -56,6 +57,14 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l
|
||||
/// cancellation while the caller's token is not cancelled ends the method as that cancellation, because it was not
|
||||
/// the caller who asked to stop.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// A socket that is being ended refuses the next call on it, and a loop that is about to use the socket at that
|
||||
/// moment fails on it instead of seeing the cancellation. The socket is ended twice over on a deliberate stop: the
|
||||
/// platform socket aborts itself as soon as the token of a call in flight is cancelled, and this shutdown aborts it
|
||||
/// again. A failure of that kind (the socket is disposed, or in an invalid state) is the stop itself and is not
|
||||
/// reported when the caller had cancelled, or when the loop had not failed before this shutdown began. Any other
|
||||
/// failure is reported whenever it arrives, even beside a cancellation.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public async Task RunAsync(Action<RealtimeEnvelope> received, CancellationToken token)
|
||||
{
|
||||
@@ -65,15 +74,27 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l
|
||||
Task ended = await Task.WhenAny(reader, writer).ConfigureAwait(false);
|
||||
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
// read before the teardown below, so a channel that failed first and //
|
||||
// was disposed a moment later still reports its failure //
|
||||
// read before the teardown below: whether the channel was disposed, //
|
||||
// so one that failed first and was disposed a moment later still //
|
||||
// reports its failure; and which loop had already failed while the //
|
||||
// caller had not cancelled, because only such a failure can be told //
|
||||
// from the socket refusing a call as it is being ended //
|
||||
////////////////////////////////////////////////////////////////////////
|
||||
bool disposed = Volatile.Read(ref _disposed) != 0;
|
||||
bool callerCancelled = token.IsCancellationRequested;
|
||||
bool readerFailedFirst = reader.IsFaulted && !callerCancelled;
|
||||
bool writerFailedFirst = writer.IsFaulted && !callerCancelled;
|
||||
Interlocked.Exchange(ref _closed, 1);
|
||||
_outgoing.Writer.TryComplete();
|
||||
await lifetime.CancelAsync().ConfigureAwait(false);
|
||||
socket.Abort();
|
||||
Exception? cause = ended.Exception?.InnerException;
|
||||
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
// a send still waiting for its acknowledgement is told why the //
|
||||
// channel closed. a refusal the run takes to be the caller's stop //
|
||||
// is not a cause, so the waiting send and the run say the same //
|
||||
///////////////////////////////////////////////////////////////////////
|
||||
Exception? cause = FailureOf(ended, !callerCancelled);
|
||||
foreach(TaskCompletionSource pending in _pending.Values)
|
||||
{
|
||||
pending.TrySetException(new RealtimeChannelClosedException("Realtime connection closed before acknowledgement; durable receipts will be replayed.", cause));
|
||||
@@ -84,11 +105,32 @@ public sealed class PrintingRealtimeChannel(WebSocket socket, RealtimeSettings l
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
// ended on purpose: disposed (whatever the loops met on a socket that //
|
||||
// was disposed under them is the dispose itself), or cancelled by the //
|
||||
// caller with neither loop having failed. anything else is rethrown: //
|
||||
// a failure, the reader's first, or a cancellation nobody asked for //
|
||||
// caller with no failure but the socket refusing a call as it ended. //
|
||||
// anything else is rethrown: a failure, the reader's first, or a //
|
||||
// cancellation nobody asked for //
|
||||
/////////////////////////////////////////////////////////////////////////
|
||||
if(!disposed && !(both.IsCanceled && token.IsCancellationRequested))
|
||||
{ await both.ConfigureAwait(false); }
|
||||
if(disposed)
|
||||
{ return; }
|
||||
Exception? failure = FailureOf(reader, readerFailedFirst) ?? FailureOf(writer, writerFailedFirst);
|
||||
if(failure is not null)
|
||||
{ ExceptionDispatchInfo.Throw(failure); }
|
||||
if(!token.IsCancellationRequested)
|
||||
{ await (reader.IsCanceled ? reader : writer.IsCanceled ? writer : both).ConfigureAwait(false); }
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// What a loop failed with, or null when it did not fail or its failure is only the stop. A socket that has been
|
||||
/// aborted refuses a call in one of two ways: it is disposed, or it is in an invalid state. A loop that met that,
|
||||
/// and is not known to have failed before the stop, was stopped by the abort and not by anything the server or the
|
||||
/// network did. A connection the server dropped fails differently (closed prematurely) and is always reported.
|
||||
/// </summary>
|
||||
private static Exception? FailureOf(Task loop, bool failedBeforeStop)
|
||||
{
|
||||
Exception? failure = loop.Exception?.InnerException;
|
||||
bool refusedByAnEndedSocket = failure is ObjectDisposedException || (failure is WebSocketException { WebSocketErrorCode: WebSocketError.InvalidState });
|
||||
return failure is null || (!failedBeforeStop && refusedByAnEndedSocket) ? null : failure;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -119,9 +119,7 @@ public sealed class ApplicationTrafficTransportTests
|
||||
finally
|
||||
{
|
||||
await timeout.CancelAsync();
|
||||
try
|
||||
{ await run; }
|
||||
catch(Exception exception) when(exception is OperationCanceledException or WebSocketException) { }
|
||||
await run;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -181,6 +181,144 @@ public sealed class RealtimeChannelEndingTests
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// The caller's cancellation can overtake the reader: the reader has checked the token and is about to ask the socket
|
||||
/// for a frame when the caller cancels and the shutdown aborts the socket, so the reader fails on the aborted socket
|
||||
/// instead of seeing the cancellation. Nothing failed, so the run still returns. The hook holds the reader at exactly
|
||||
/// that point until the socket has been aborted.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task CallerCancellationThatOvertakesTheReaderStillEndsTheRunNormally()
|
||||
{
|
||||
using var stop = new CancellationTokenSource();
|
||||
using var socket = new ScriptedWebSocket();
|
||||
int receives = 0;
|
||||
socket.Receiving = () =>
|
||||
{
|
||||
if(Interlocked.Increment(ref receives) == 2)
|
||||
{
|
||||
stop.Cancel();
|
||||
Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience));
|
||||
}
|
||||
};
|
||||
using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings());
|
||||
Task run = channel.RunAsync(_ => { }, stop.Token);
|
||||
socket.Deliver("PONG", "pong-1", new { serverTime = "2026-01-01T00:00:00Z" });
|
||||
await run.WaitAsync(Patience);
|
||||
Assert.True(run.IsCompletedSuccessfully);
|
||||
Assert.Equal(2, Volatile.Read(ref receives));
|
||||
Assert.Equal(1, socket.Aborts);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// What a loop meets after the shutdown has aborted the socket decides whether the run fails. The writer is held in
|
||||
/// its send until the caller has cancelled and the socket has been aborted, and then fails. A socket that refuses the
|
||||
/// call because it has ended (disposed, or in an invalid state) is the stop itself, so the run returns. Anything
|
||||
/// else is a failure and is reported although it arrived after the cancellation: an input or output error, and a
|
||||
/// connection the server dropped, which the socket reports as closed prematurely.
|
||||
/// </summary>
|
||||
[Theory]
|
||||
[InlineData("disposed", false)]
|
||||
[InlineData("invalid-state", false)]
|
||||
[InlineData("closed-prematurely", true)]
|
||||
[InlineData("io", true)]
|
||||
public async Task FailureMetAfterTheSocketWasAbortedIsReportedUnlessItIsTheSocketRefusingTheCall(string kind, bool reported)
|
||||
{
|
||||
Exception met = kind switch
|
||||
{
|
||||
"disposed" => new ObjectDisposedException("socket"),
|
||||
"invalid-state" => new WebSocketException(WebSocketError.InvalidState),
|
||||
"closed-prematurely" => new WebSocketException(WebSocketError.ConnectionClosedPrematurely),
|
||||
_ => new IOException("Synthetic send failure."),
|
||||
};
|
||||
using var stop = new CancellationTokenSource();
|
||||
using var socket = new ScriptedWebSocket();
|
||||
socket.Sending = _ =>
|
||||
{
|
||||
stop.Cancel();
|
||||
Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience));
|
||||
throw met;
|
||||
};
|
||||
using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings());
|
||||
Task run = channel.RunAsync(_ => { }, stop.Token);
|
||||
await channel.SendAsync("HEARTBEAT", new { });
|
||||
if(reported)
|
||||
{
|
||||
Assert.Same(met, await Assert.ThrowsAnyAsync<Exception>(() => run.WaitAsync(Patience)));
|
||||
}
|
||||
else
|
||||
{
|
||||
await run.WaitAsync(Patience);
|
||||
Assert.True(run.IsCompletedSuccessfully);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// The platform socket aborts itself as soon as the token of a call in flight is cancelled, before this channel's own
|
||||
/// shutdown begins, and from then on refuses a new call as being in an invalid state. A writer that is about to send
|
||||
/// at that moment therefore fails before the shutdown has looked at the loops. The reader is held in its callback
|
||||
/// until the shutdown aborts the socket, which it does only after a loop has ended, so the writer is the loop that
|
||||
/// ended first and had already failed when the shutdown looked. The caller had cancelled, so the refusal is still
|
||||
/// the stop and the run returns.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task SocketRefusingACallBecauseTheCallerCancelledIsTheStopEvenBeforeTheShutdownBegins()
|
||||
{
|
||||
using var stop = new CancellationTokenSource();
|
||||
using var socket = new ScriptedWebSocket();
|
||||
socket.Sending = _ =>
|
||||
{
|
||||
stop.Cancel();
|
||||
throw new WebSocketException(WebSocketError.InvalidState);
|
||||
};
|
||||
using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings());
|
||||
Task run = channel.RunAsync(_ => Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience)), stop.Token);
|
||||
socket.Deliver("PONG", "pong-1", new { serverTime = "2026-01-01T00:00:00Z" });
|
||||
await channel.SendAsync("HEARTBEAT", new { });
|
||||
await run.WaitAsync(Patience);
|
||||
Assert.True(run.IsCompletedSuccessfully);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Without a cancellation by the caller, a loop is taken to have been stopped by the shutdown only when it had not
|
||||
/// failed before the shutdown began. Nobody cancels here: the callback gives up, as the service's does when a newer
|
||||
/// connection has replaced this one, while the writer is inside a send. The shutdown then aborts the socket, and the
|
||||
/// writer's send fails on it. That refusal came after the shutdown began, so it is the shutdown itself, and the run
|
||||
/// ends as the cancellation the callback raised and not as the socket's refusal.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task RefusalMetAfterTheShutdownBeganIsNotReportedAlthoughNobodyCancelled()
|
||||
{
|
||||
using var never = new CancellationTokenSource();
|
||||
using var socket = new ScriptedWebSocket();
|
||||
using var sending = new ManualResetEventSlim();
|
||||
socket.Sending = _ =>
|
||||
{
|
||||
sending.Set();
|
||||
Assert.True(SpinWait.SpinUntil(() => socket.Aborts > 0, Patience));
|
||||
return Task.CompletedTask;
|
||||
};
|
||||
using var channel = new PrintingRealtimeChannel(socket, new RealtimeSettings());
|
||||
Task run = channel.RunAsync(_ =>
|
||||
{
|
||||
Assert.True(sending.Wait(Patience));
|
||||
throw new OperationCanceledException("Replaced by a newer connection.");
|
||||
}, never.Token);
|
||||
await channel.SendAsync("HEARTBEAT", new { });
|
||||
socket.Deliver("JOB_AVAILABLE", "wake-1", new { });
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(() => run.WaitAsync(Patience));
|
||||
Assert.True(run.IsCanceled);
|
||||
Assert.False(never.IsCancellationRequested);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// <summary>A failure is not hidden by a cancellation that arrives with it: when the socket refuses a frame just as the caller cancels, the run still ends with the failure.</summary>
|
||||
[Fact]
|
||||
public async Task FailureIsReportedEvenWhenTheCallerCancelsAtTheSameMoment()
|
||||
|
||||
@@ -29,6 +29,11 @@ internal sealed class ScriptedWebSocket : WebSocket
|
||||
|
||||
|
||||
|
||||
/// <summary>Called each time the channel asks for a frame, before the socket looks at its own state; a test uses it to act at the moment the reader is about to receive.</summary>
|
||||
internal Action? Receiving { get; set; }
|
||||
|
||||
|
||||
|
||||
/// <summary>Called for every frame the channel sends, before it is recorded; a test uses it to stall or fail the send.</summary>
|
||||
internal Func<CancellationToken, Task>? Sending { get; set; }
|
||||
|
||||
@@ -166,6 +171,7 @@ internal sealed class ScriptedWebSocket : WebSocket
|
||||
/// </summary>
|
||||
public override async Task<WebSocketReceiveResult> ReceiveAsync(ArraySegment<byte> buffer, CancellationToken cancellationToken)
|
||||
{
|
||||
Receiving?.Invoke();
|
||||
ThrowIfEnded();
|
||||
if(!await _incoming.Reader.WaitToReadAsync(cancellationToken))
|
||||
{ throw new WebSocketException(WebSocketError.ConnectionClosedPrematurely); }
|
||||
|
||||
Reference in new issue
Block a user