using System.Collections.Frozen;
using System.Globalization;
using System.Net.WebSockets;
using System.Security.Claims;
using System.Text.Json;
using DodoSSH.Api.Authorization;
using DodoSSH.Api.Setup;
using DodoSSH.Contracts;
using DodoSSH.Domain.Authorization;
using FastEndpoints;
using Microsoft.Extensions.Options;
namespace DodoSSH.Api.Features.Events;
///
/// The socket that says "pull now" so a client does not have to wait for its timer.
///
///
///
/// Everything this endpoint sends is a notice. It never carries an item, a payload or a
/// cursor: the client's answer to a notice is the delta pull it would have run on its own anyway, so
/// there is exactly one code path that applies a change and this is not it. See ADR 0012 for why
/// pushing the items themselves is refused.
///
///
/// The bearer token authorises the upgrade, unlike the relay's ticket in ADR 0004. The relay's socket
/// is a byte pipe whose whole authorization decision is made before it opens; this one is a view of
/// the caller's own vault list and has to keep answering "what may this account read" for as long as
/// it is held. Its two bounds on that — the token's own expiry, and a periodic re-resolve — are in
/// .
///
///
internal sealed class VaultEventsEndpoint(
IServiceScopeFactory scopes,
VaultEventHub hub,
IOptions options,
TimeProvider clock,
IHostApplicationLifetime lifetime,
ILogger logger)
: EndpointWithoutRequest
{
///
/// The largest message this endpoint will read from a client.
///
///
/// A client sends nothing but ping, so the cap is three orders of magnitude of headroom and
/// still small enough that a hostile client cannot make the server buffer anything worth having.
///
private const int MaxInboundFrameBytes = 4 * 1024;
/// How long to wait for the close handshake before dropping the socket.
private static readonly TimeSpan CloseTimeout = TimeSpan.FromSeconds(5);
///
public override void Configure()
{
Get(VaultEvents.Path);
// Enrolled, matching sync. A caller with no identity key holds no vault key either, so every
// notice this socket could send is about ciphertext they cannot read.
Policies(Auth.EnrolledPolicy);
Description(b => b
.WithName("VaultEvents")
.WithSummary("Pushes a notice when a vault the caller can read has changed.")
.WithTags("Events"));
}
///
public override async Task HandleAsync(CancellationToken ct)
{
if (await RefusedAsync().ConfigureAwait(false))
{
return;
}
var (userId, vaults) = await ResolveAccessAsync(ct).ConfigureAwait(false);
// Admitted before the upgrade so a refusal costs nothing, but answered *through* the socket
// rather than as an HTTP status: a constrained WebSocket client cannot read the status of a
// failed upgrade, and "you have too many open" is precisely the case where the client needs
// to know to back off rather than retry. Same reasoning as ADR 0004 on request headers.
var connection = hub.TryAdmit(userId, vaults);
try
{
using var socket = await HttpContext.WebSockets
.AcceptWebSocketAsync(new WebSocketAcceptContext { SubProtocol = VaultEvents.SubProtocol })
.ConfigureAwait(false);
if (connection is null)
{
await CloseAsync(
socket,
new Closure(
VaultEvents.TooManyConnectionsCloseCode, "Too many open event sockets."))
.ConfigureAwait(false);
return;
}
await PumpAsync(socket, connection, ct).ConfigureAwait(false);
}
finally
{
// Inside a try that starts *before* the upgrade, because an accept that throws — a client
// that abandoned the handshake — would otherwise leave an admitted connection in the hub
// for the life of the process, counting against this account's cap and taking a slot from
// the sockets that did open.
if (connection is not null)
{
hub.Remove(connection);
}
}
}
///
/// Answers the requests that are not an event socket at all, as ordinary HTTP.
///
/// Whether a response was sent and the handler should stop.
///
/// All three answers are problem documents rather than bare statuses, because each one has a
/// different remedy and a client that cannot tell them apart would retry the two that will never
/// succeed. Answered before the upgrade, so a caller that got the handshake wrong reads why in a
/// body rather than inferring it from a socket that closed.
///
private async Task RefusedAsync()
{
if (!options.Value.Enabled)
{
// 404 rather than 501: the feature is absent from this deployment, and /api/v1/meta does
// not advertise it. A client that dialled anyway keeps polling, which is correct.
await Send.ResultAsync(Problems.Coded(
StatusCodes.Status404NotFound,
ProblemCodes.EventsUnavailable,
"This server does not push vault changes. Synchronise on a timer instead; "
+ "GET /api/v1/meta lists the features it does offer."))
.ConfigureAwait(false);
return true;
}
if (!HttpContext.WebSockets.IsWebSocketRequest)
{
await Send.ResultAsync(Problems.Coded(
StatusCodes.Status400BadRequest,
ProblemCodes.MalformedRequest,
"This endpoint is a WebSocket. Send an upgrade request offering the "
+ $"'{VaultEvents.SubProtocol}' subprotocol."))
.ConfigureAwait(false);
return true;
}
// The subprotocol is this API's version negotiation for the socket, so an upgrade that does
// not offer it is refused rather than accepted and answered in a dialect the caller may not
// read. See VaultEvents.SubProtocol.
if (!HttpContext.WebSockets.WebSocketRequestedProtocols
.Contains(VaultEvents.SubProtocol, StringComparer.Ordinal))
{
await Send.ResultAsync(Problems.Coded(
StatusCodes.Status400BadRequest,
ProblemCodes.MalformedRequest,
$"This server speaks '{VaultEvents.SubProtocol}', which the upgrade request did "
+ "not offer."))
.ConfigureAwait(false);
return true;
}
return false;
}
///
/// Reads who the caller is and which vaults they may follow, in a scope of its own.
///
///
/// A fresh scope, disposed at once, rather than services injected into this endpoint — which is
/// the lesson ADR 0004 records paying for on the relay. This handler runs for as long as the
/// socket is open, so anything scoped it held would be a DbContext alive for hours, and a
/// few hundred of those exhaust the connection pool. The database is touched here and in
/// , briefly, and nowhere else.
///
private async Task<(Guid UserId, FrozenSet Vaults)> ResolveAccessAsync(
CancellationToken cancellationToken)
{
var scope = scopes.CreateAsyncScope();
await using var _ = scope.ConfigureAwait(false);
var currentUser = scope.ServiceProvider.GetRequiredService();
var vaultAccess = scope.ServiceProvider.GetRequiredService();
var user = await currentUser.GetOrProvisionAsync(cancellationToken).ConfigureAwait(false);
return (user.Id, await ReachableAsync(vaultAccess, user.Id, cancellationToken).ConfigureAwait(false));
}
private static async Task> ReachableAsync(
IVaultAccessService vaultAccess,
Guid userId,
CancellationToken cancellationToken)
{
var accessible = await vaultAccess.ListAsync(userId, cancellationToken).ConfigureAwait(false);
return accessible
.Where(access => access.Vault is not null
&& access.Permissions.HasFlag(PermissionFlags.Read))
.Select(access => access.Vault!.Id)
.ToFrozenSet();
}
/// Runs the socket until something ends it, then closes it politely.
///
///
/// Three loops rather than one: reading a socket and writing to it are independent waits, and the
/// clock is a third. They are joined by , and the sender is
/// finished before the close is written, because a close frame racing a notice frame is a protocol
/// violation that presents as a client dropping its connection for no visible reason.
///
///
/// No operation on the WebSocket is ever cancelled. Cancelling a pending receive or send on
/// the socket Kestrel hands over aborts it, and an aborted socket cannot send the close frame that
/// is the only way a client learns why it was dropped — 4401 "reconnect with a fresh
/// token" against 1001 "the server is going away" against a failure to back off from. This was
/// once the case here, and it went unnoticed because TestServer's in-memory socket survives
/// the cancellation that a real one does not. So the token stops the queue and the clock; the
/// sender is let finish its last write; the close goes out; and the receiver ends on the client's
/// answering close. Each wait is bounded, and the socket is aborted only when a peer has stopped
/// answering altogether. The relay's pump does the same, for the same reason.
///
///
private async Task PumpAsync(
WebSocket socket,
VaultEventConnection connection,
CancellationToken requestAborted)
{
var settings = options.Value;
using var pump = CancellationTokenSource.CreateLinkedTokenSource(
requestAborted, lifetime.ApplicationStopping);
var closure = new Closure(
(int)WebSocketCloseStatus.NormalClosure, string.Empty);
connection.TryEnqueue(new VaultEvent(
VaultEventKinds.Hello,
ServerTime: clock.GetUtcNow(),
HeartbeatSeconds: (int)settings.HeartbeatInterval.TotalSeconds,
VaultCount: connection.VaultCount));
var sending = SendAsync(socket, connection, pump.Token);
var receiving = ReceiveAsync(socket, connection);
var minding = MindAsync(connection, closure, pump.Token);
await Task.WhenAny(sending, receiving, minding).ConfigureAwait(false);
await pump.CancelAsync().ConfigureAwait(false);
// Nothing may still be mid-send when the close frame goes out. A send that does not finish is
// a client that has stopped reading, which will not read a close frame either.
if (!await SettlesAsync(sending).ConfigureAwait(false))
{
socket.Abort();
}
await SettlesAsync(minding).ConfigureAwait(false);
if (lifetime.ApplicationStopping.IsCancellationRequested)
{
// 1001 "going away", so a client knows to reconnect immediately rather than treating a
// rolling deployment as a server that has broken.
closure.Set((int)WebSocketCloseStatus.EndpointUnavailable, "The server is shutting down.");
}
EventsLog.ClosingConnection(logger, connection.UserId, closure.Reason);
await CloseAsync(socket, closure).ConfigureAwait(false);
// The client's answering close is what ends the receiver. Returning before it arrives would
// dispose the socket under a close frame still in flight.
if (!await SettlesAsync(receiving).ConfigureAwait(false))
{
socket.Abort();
await SettlesAsync(receiving).ConfigureAwait(false);
}
}
///
/// Writes queued frames to the socket, one at a time.
///
///
/// The only writer, which is what makes concurrent sends impossible without a lock: the
/// heartbeat and the pong both go into the same queue rather than to the socket. A WebSocket
/// permits one send at a time and faults permanently on a second, so this is not a tidiness
/// preference.
///
private async Task SendAsync(
WebSocket socket,
VaultEventConnection connection,
CancellationToken cancellationToken)
{
await foreach (var frame in connection.Outbound
.ReadAllAsync(cancellationToken)
.ConfigureAwait(false))
{
// Re-resolved before the notice is forwarded, not after: the client's answer to this frame
// is to re-read its vault list, and the point of a newly shared vault is that the *next*
// change to it produces a notice too. A socket that forwarded first would not follow the
// new vault until its next periodic refresh.
if (string.Equals(frame.Kind, VaultEventKinds.VaultsChanged, StringComparison.Ordinal))
{
await RefreshAsync(connection, cancellationToken).ConfigureAwait(false);
}
var bytes = JsonSerializer.SerializeToUtf8Bytes(frame, DodoSshJsonContext.Default.VaultEvent);
// Not cancellable, deliberately. See PumpAsync.
await socket
.SendAsync(bytes, WebSocketMessageType.Text, endOfMessage: true, CancellationToken.None)
.ConfigureAwait(false);
}
}
///
/// Reads what the client sends, which in this version is heartbeats and a close.
///
///
///
/// A client cannot ask to follow a vault, and that is deliberate rather than unfinished: a
/// subscribe(vaultId) frame is an existence oracle for vault ids, which is the disclosure
/// SyncPullEndpoint answers 404 rather than 403 to avoid. Subscription is decided from the
/// caller's access and nothing else.
///
///
/// It takes no token: it ends on the client's close, a frame it refuses, or the socket failing —
/// and when the server is the one ending the connection, the close it sends is what prompts the
/// first of those. See .
///
///
private static async Task ReceiveAsync(WebSocket socket, VaultEventConnection connection)
{
var buffer = new byte[MaxInboundFrameBytes];
while (true)
{
// Not cancellable, deliberately. See PumpAsync.
var received = await socket.ReceiveAsync(buffer, CancellationToken.None).ConfigureAwait(false);
if (received.MessageType == WebSocketMessageType.Close)
{
return;
}
// Oversized, or split across frames. Nothing this protocol sends is either, so the client
// is broken or probing; ending the socket is cheaper than reassembling for it.
if (!received.EndOfMessage)
{
return;
}
// Binary is unused in v1 and skipped rather than refused, because ADR 0012 reserves it for
// shared-session data — an older server meeting a newer client must ignore those, not
// close on them.
if (received.MessageType != WebSocketMessageType.Text)
{
continue;
}
if (Kind(buffer.AsSpan(0, received.Count)) is VaultEventKinds.Ping)
{
connection.TryEnqueue(new VaultEvent(VaultEventKinds.Pong));
}
}
}
///
/// Reads a frame's kind, or null if it is not one this server understands.
///
///
/// A frame that will not parse is skipped rather than closing the socket. This is a control
/// channel whose failure mode is "the client polls instead", so tolerating a frame from a newer
/// client costs nothing and refusing one costs that client its push for the whole session.
///
private static string? Kind(ReadOnlySpan utf8)
{
try
{
return JsonSerializer.Deserialize(utf8, DodoSshJsonContext.Default.VaultEvent)?.Kind;
}
catch (JsonException)
{
return null;
}
}
///
/// Keeps the heartbeat going, the vault set current, and the socket inside its token's lifetime.
///
///
///
/// The deadline is the earlier of the access token's exp and a hard cap on how long any one
/// socket may live. Closing on expiry is what keeps a long-lived connection from outliving the
/// short-lived credential that authorised it; the client answers by reconnecting with a fresh
/// token, which is a sub-second gap in a channel that degrades to polling anyway.
///
///
/// The wait is the shorter of the heartbeat and the time left, so the deadline is met to within a
/// tick rather than to within a heartbeat.
///
///
private async Task MindAsync(
VaultEventConnection connection,
Closure closure,
CancellationToken cancellationToken)
{
var settings = options.Value;
var started = clock.GetUtcNow();
var deadline = TokenExpiry() is { } expiry && expiry < started + settings.MaxConnectionDuration
? (Expiry: expiry, ForToken: true)
: (Expiry: started + settings.MaxConnectionDuration, ForToken: false);
var refreshed = started;
while (!cancellationToken.IsCancellationRequested)
{
var now = clock.GetUtcNow();
var remaining = deadline.Expiry - now;
if (remaining <= TimeSpan.Zero)
{
closure.Set(
deadline.ForToken
? VaultEvents.TokenExpiredCloseCode
: (int)WebSocketCloseStatus.NormalClosure,
deadline.ForToken
? "The access token has expired. Reconnect with a fresh one."
: "This connection reached its maximum lifetime.");
return;
}
var wait = settings.HeartbeatInterval < remaining ? settings.HeartbeatInterval : remaining;
await Task.Delay(wait, clock, cancellationToken).ConfigureAwait(false);
now = clock.GetUtcNow();
if (now - refreshed >= settings.AccessRefreshInterval)
{
// The backstop for a grant withdrawn while this socket was open. What it bounds is
// metadata — that a vault changed — because that is all a notice carries and reading
// the vault still needs a key this server has never held. See ADR 0012.
await RefreshAsync(connection, cancellationToken).ConfigureAwait(false);
refreshed = now;
}
connection.TryEnqueue(new VaultEvent(VaultEventKinds.Ping, ServerTime: now));
}
}
/// Re-reads which vaults this socket may follow.
///
/// A failure is logged and swallowed. The alternative is dropping a working socket because one
/// database call timed out, which would trade an occasionally stale vault set for an outage.
///
private async Task RefreshAsync(VaultEventConnection connection, CancellationToken cancellationToken)
{
try
{
var scope = scopes.CreateAsyncScope();
await using var _ = scope.ConfigureAwait(false);
var vaultAccess = scope.ServiceProvider.GetRequiredService();
connection.Resubscribe(
await ReachableAsync(vaultAccess, connection.UserId, cancellationToken)
.ConfigureAwait(false));
}
catch (OperationCanceledException)
{
// The socket is closing.
}
catch (Exception exception) when (exception is not OutOfMemoryException)
{
EventsLog.AccessRefreshFailed(logger, connection.UserId, exception);
}
}
///
/// MapInboundClaims is off — see — so the claim is spelled as the
/// provider issued it rather than as a WS-Federation URI. Null is treated as "no bound from the
/// token", which the bearer handler's RequireExpirationTime should make unreachable; the
/// lifetime cap covers it either way.
///
private DateTimeOffset? TokenExpiry() =>
long.TryParse(
HttpContext.User.FindFirstValue("exp"),
NumberStyles.Integer,
CultureInfo.InvariantCulture,
out var seconds)
? DateTimeOffset.FromUnixTimeSeconds(seconds)
: null;
private static async Task CloseAsync(WebSocket socket, Closure closure)
{
if (socket.State is not (WebSocketState.Open or WebSocketState.CloseReceived))
{
return;
}
using var timeout = new CancellationTokenSource(CloseTimeout);
try
{
await socket
.CloseOutputAsync((WebSocketCloseStatus)closure.Code, closure.Reason, timeout.Token)
.ConfigureAwait(false);
}
catch (Exception exception)
when (exception is OperationCanceledException or WebSocketException or ObjectDisposedException)
{
// The peer is already gone. There is nothing to tell it and nothing to recover.
}
}
///
/// Waits for a pump loop to finish, for at most the close timeout.
///
/// Whether it finished.
///
/// Every one of these loops ends by being cancelled or by the socket going away, so an exception
/// here is the expected shape of "this connection is over" rather than a fault to propagate — and
/// propagating it would skip the close frame the other side is waiting for.
///
private static async Task SettlesAsync(Task loop)
{
try
{
await loop.WaitAsync(CloseTimeout).ConfigureAwait(false);
return true;
}
catch (TimeoutException)
{
return false;
}
catch (Exception exception)
when (exception is OperationCanceledException or WebSocketException or ObjectDisposedException)
{
return true;
}
}
/// Why the socket is being closed, decided by whichever loop ended first.
///
/// Mutable and shared, and safe without a lock for one specific reason: it is written only by
/// , and read only after that loop has been awaited, which is a memory
/// barrier. If that loop outlived the close timeout, a late write races the read, and either
/// answer is still a true one.
///
private sealed class Closure(int code, string reason)
{
internal int Code { get; private set; } = code;
internal string Reason { get; private set; } = reason;
internal void Set(int code, string reason)
{
Code = code;
Reason = reason;
}
}
}