using System.Globalization;
using System.Runtime.InteropServices;
using DodoSSH.Client.Api;
using DodoSSH.Client.Storage;
using DodoSSH.Contracts;
namespace DodoSSH.Client.Sync;
///
/// One vault's synchronisation pass: pull, reconcile, push, pull again.
///
///
///
/// Pull first, so a local change is merged against the newest server state before it is offered — which
/// turns most would-be conflicts into ordinary merges and keeps the push round count down. Push second.
/// Pull once more at the end only if something was pushed, so the mirror reflects the versions the
/// server actually assigned.
///
///
/// Pulling does not decrypt. A change with no local work pending is copied into the mirror as
/// ciphertext and nothing more. Decryption happens when a merge needs it, or when the interface reads an
/// item. For a five-thousand-item first sync that is the difference between plumbing bytes and running
/// ten thousand AEAD operations for nothing.
///
///
public sealed class SyncEngine
{
private readonly ISyncApi api;
private readonly ItemStore items;
private readonly OutboxStore outbox;
private readonly SyncStateStore syncState;
private readonly ConflictStore conflicts;
private readonly VaultKeyring keyring;
private readonly TimeProvider clock;
private readonly SyncOptions options;
///
/// One per synchronised item type, built once. The keys are also the pull filter — see
/// — so a type this engine cannot reconcile is never requested, and a type it
/// can reconcile cannot be left out of the request.
///
private readonly Dictionary reconcilers;
/// Creates the engine.
public SyncEngine(
ISyncApi api,
ItemStore items,
OutboxStore outbox,
SyncStateStore syncState,
ConflictStore conflicts,
VaultKeyring keyring,
TimeProvider clock,
SyncOptions? options = null)
{
ArgumentNullException.ThrowIfNull(api);
ArgumentNullException.ThrowIfNull(items);
ArgumentNullException.ThrowIfNull(outbox);
ArgumentNullException.ThrowIfNull(syncState);
ArgumentNullException.ThrowIfNull(conflicts);
ArgumentNullException.ThrowIfNull(keyring);
ArgumentNullException.ThrowIfNull(clock);
this.api = api;
this.items = items;
this.outbox = outbox;
this.syncState = syncState;
this.conflicts = conflicts;
this.keyring = keyring;
this.clock = clock;
this.options = options ?? SyncOptions.Default;
reconcilers = ItemKinds.Reconcilers(outbox, conflicts, keyring);
}
/// Runs a full pass over one vault.
public async Task SyncAsync(Guid vaultId, CancellationToken cancellationToken)
{
var report = new SyncReportBuilder(vaultId);
await PullAsync(vaultId, report, cancellationToken).ConfigureAwait(false);
var pushedAnything = false;
for (var round = 1; ; round++)
{
var outcome = await DrainAsync(vaultId, report, cancellationToken).ConfigureAwait(false);
if (outcome.Sent == 0)
{
break;
}
pushedAnything = true;
if (!outcome.NeedsAnotherRound)
{
break;
}
if (round >= options.MaxPushRounds)
{
report.RoundsExhausted = true;
break;
}
}
if (pushedAnything)
{
await PullAsync(vaultId, report, cancellationToken).ConfigureAwait(false);
}
return report.Build();
}
///
/// Reads every change available and applies it.
///
///
/// The cursor is saved after each page's changes are applied, which makes applying at-least-once
/// rather than exactly-once: a process that dies between the two re-reads that page next time. That
/// is deliberate and safe, because applying a change is a blind overwrite of a mirror row and a
/// resurrection takes a deterministic id. The other ordering — save the cursor first — would lose
/// changes outright, which no amount of idempotence can repair.
///
private async Task PullAsync(
Guid vaultId,
SyncReportBuilder report,
CancellationToken cancellationToken)
{
var state = await syncState.ReadAsync(vaultId, cancellationToken).ConfigureAwait(false);
// At most one restart per pull. A server that rejects the cursor it has just issued is not
// telling this client anything it can act on, and replaying the whole log against it would turn
// one broken deployment into an unbounded amount of work.
var alreadyStartedOver = false;
for (var page = 0; page < options.MaxPullPages; page++)
{
SyncPullResponse response;
try
{
response = await api.SyncPullAsync(
vaultId,
new SyncPullRequest(state.Cursor, options.PullPageSize, ItemKinds.SyncedTypes),
cancellationToken).ConfigureAwait(false);
}
catch (DodoSshApiException exception)
when (!alreadyStartedOver && WasTheCursorRefused(exception, state.Cursor))
{
alreadyStartedOver = true;
state = await StartOverAsync(state, report, cancellationToken).ConfigureAwait(false);
continue;
}
foreach (var change in response.Changes)
{
await ApplyAsync(vaultId, change, report, cancellationToken).ConfigureAwait(false);
report.Pulled++;
// Counted apart for the same reason the pushed ones are: a machine reads back the log
// entries it just wrote, so a pass that "pulled five changes" may have carried nothing
// anybody did. See SyncReport.PulledItems.
if (IsLog(change.EntityType))
{
report.PulledLogEntries++;
}
}
var advanced = !string.Equals(state.Cursor, response.NextCursor, StringComparison.Ordinal);
state = Record(vaultId, state, response, report);
await syncState.SaveAsync(state, cancellationToken).ConfigureAwait(false);
if (!response.HasMore)
{
break;
}
// A server that claims more but neither returns a change nor moves the cursor would spin
// this loop for ever. Stopping is the only safe reading of that answer.
if (!advanced && response.Changes.Count == 0)
{
break;
}
}
}
///
/// Whether the server refused the cursor this client sent, rather than failing for some other reason.
///
///
/// Read from the problem code and not from the prose, which is free to change, and never claimed for a
/// request that carried no cursor: "from the beginning" is the one position a client is allowed to ask
/// for, so a rejection of that is a server this code cannot reason about and has to surface.
/// It is also what keeps the retry from looping — the restarted request sends no cursor.
///
/// Whether a change is one the machine wrote about itself rather than one somebody made.
///
/// Used only for reporting. Log entries sync exactly like every other item — they are pulled, pushed,
/// merged and stored by the same code — and this distinguishes them nowhere except in the two numbers a
/// person reads.
///
private static bool IsLog(SyncEntityType type) =>
type is SyncEntityType.ConnectionLogEntry or SyncEntityType.ActivityLogEntry;
private static bool WasTheCursorRefused(DodoSshApiException exception, string? cursor) =>
!string.IsNullOrEmpty(cursor)
&& string.Equals(exception.Code, ProblemCodes.InvalidCursor, StringComparison.Ordinal);
///
/// Forgets this vault's position and reads the log again from the beginning.
///
///
///
/// A cursor the server will not accept is not a transient failure. Every later pass reads the same
/// stored cursor and is refused the same way, so a vault that met one stayed there for good — pulling
/// nothing, and pushing nothing either, because the pass threw before it reached the outbox. The user
/// saw a 400 saying to resync from the beginning and had no way to do it. This is that way.
///
///
/// The causes are all on the far side: a rotated cursor signing key, a vault served from a restored
/// database whose sequences no longer reach that far, a cache copied between machines. None of them is
/// something the person at the keyboard did, and none of them is something they could act on if asked.
///
///
/// The mirror is deliberately kept. Replaying from the beginning rewrites every row the server
/// still has — applying a change is a blind overwrite — so the re-pull repairs the mirror as it goes.
/// Clearing it first would claim more than the evidence supports: the position was refused, not the
/// contents, and a machine that loses its connection halfway through the replay would be left with
/// less than it started with. SyncStateStore.ResetAsync is the heavier remedy, for when the
/// cache itself is the thing in doubt.
///
///
/// That leaves one gap, and it is worth naming rather than leaving to be discovered: once the server
/// starts collecting tombstones — DodoOptions.TombstoneRetentionDays, not implemented yet — a
/// replay no longer carries a deletion older than the retention window. A machine that missed such a
/// delete and then had its cursor refused would keep the row. Nothing here can tell that from a row
/// the server still has, so the answer when it matters will be to clear the mirror as well, not to
/// guess.
///
///
/// Written down before the replay begins, so a process that dies mid-replay starts the next one from
/// the beginning as well, rather than meeting the same refusal again.
///
///
private async Task StartOverAsync(
StoredSyncState state,
SyncReportBuilder report,
CancellationToken cancellationToken)
{
var restarted = state with { Cursor = null };
await syncState.SaveAsync(restarted, cancellationToken).ConfigureAwait(false);
report.ResyncedFromStart = true;
return restarted;
}
private StoredSyncState Record(
Guid vaultId,
StoredSyncState state,
SyncPullResponse response,
SyncReportBuilder report)
{
var now = clock.GetUtcNow();
var skew = (long)(response.ServerTime - now).TotalMilliseconds;
report.ServerKeyGeneration = response.CurrentKeyGeneration;
report.ServerTimeSkewMs = skew;
// A generation ahead of the key this client holds means the vault was rekeyed and this client's
// grant has not been re-wrapped. Items pulled meanwhile are stored but cannot be read.
keyring.TryGet(vaultId, out _, out var held);
report.RekeyRequired = response.CurrentKeyGeneration > held;
return state with
{
Cursor = response.NextCursor,
KeyGeneration = response.CurrentKeyGeneration,
LastPulledAt = now,
ServerTimeSkewMs = skew,
};
}
private async Task ApplyAsync(
Guid vaultId,
SyncChange change,
SyncReportBuilder report,
CancellationToken cancellationToken)
{
if (!reconcilers.TryGetValue(change.EntityType, out var reconciler))
{
// Reserved in the contract but not yet syncable here. The pull filter already asks for only
// the types this build handles, so reaching this means a newer server sent something extra —
// and ignoring it keeps that from breaking an older client's pull. Not mirrored either: a row
// this build can never read is cache with no reader.
return;
}
await ItemMirror.WriteAsync(items, vaultId, change, cancellationToken).ConfigureAwait(false);
var pending = await outbox
.FindAsync(vaultId, change.EntityType, change.EntityId, cancellationToken)
.ConfigureAwait(false);
if (pending is null || pending.IsParked)
{
return;
}
await reconciler.ReconcileAsync(vaultId, change, pending, report, cancellationToken)
.ConfigureAwait(false);
}
/// What one push round achieved.
[StructLayout(LayoutKind.Auto)]
private readonly record struct DrainOutcome(int Sent, int Conflicts, bool BatchWasFull)
{
internal bool NeedsAnotherRound => Conflicts > 0 || BatchWasFull;
}
///
/// Sends one batch and acts on each per-operation answer.
///
///
/// The cursor in the push response is deliberately ignored. It sits after this push's own
/// changes, so adopting it would skip any change another client committed at a lower sequence
/// between this client's last pull and this push — permanently. Continuing from the cursor this
/// client already holds re-reads its own writes, which costs one redundant page and is idempotent.
///
private async Task DrainAsync(
Guid vaultId,
SyncReportBuilder report,
CancellationToken cancellationToken)
{
var pending = await outbox
.TakeAsync(vaultId, options.MaxOperationsPerPush, cancellationToken)
.ConfigureAwait(false);
if (pending.Count == 0)
{
return default;
}
var operations = new List(pending.Count);
var byOperationId = new Dictionary(pending.Count);
foreach (var operation in pending)
{
var payload = await CurrentAsync(vaultId, operation, cancellationToken)
.ConfigureAwait(false);
operations.Add(new SyncPushOperation(
operation.OperationId,
operation.EntityType,
operation.EntityId,
operation.Operation,
operation.ExpectedVersion,
payload,
operation.Fields));
byOperationId[operation.OperationId] = operation;
await outbox.MarkDispatchedAsync(operation.Sequence, cancellationToken)
.ConfigureAwait(false);
}
var response = await api
.SyncPushAsync(vaultId, new SyncPushRequest(operations), cancellationToken)
.ConfigureAwait(false);
var conflicted = 0;
foreach (var result in response.Results)
{
if (!byOperationId.TryGetValue(result.OperationId, out var operation))
{
// An id this client did not send. Nothing sane to do with it.
continue;
}
// The attempt count was incremented above, so the bound is read from the fresh value.
var attempts = operation.Attempts + 1;
if (await HandleAsync(vaultId, operation with { Attempts = attempts }, result, report, cancellationToken)
.ConfigureAwait(false))
{
conflicted++;
}
}
return new DrainOutcome(
pending.Count, conflicted, pending.Count == options.MaxOperationsPerPush);
}
///
/// The payload to send, re-sealed under the current key if it was queued before a rotation.
///
///
///
/// Nothing may reach the server under a superseded generation, and this is where that is
/// enforced. A change queued offline is sealed under whatever key was current when it was made,
/// and a rotation can land in between — so sending it as it stands would store a brand-new item
/// under the key the person who was just removed still holds. The vault's own re-sealing pass cannot
/// help: it runs over what the server holds, and this has not been sent yet.
///
///
/// The revised payload is written back to the outbox before it goes, so a push whose answer is lost
/// is retried as the same bytes. Re-sealing on each attempt instead would produce a different
/// envelope every time, which is harmless on the wire and would leave the queued row disagreeing
/// with what the server may already have accepted.
///
///
private async Task CurrentAsync(
Guid vaultId,
PendingOperation operation,
CancellationToken cancellationToken)
{
if (operation.Payload is not { } payload
|| !keyring.TryGet(vaultId, out _, out var generation)
|| payload.KeyGeneration >= generation)
{
return operation.Payload;
}
var version = SyncVersions.NextVersion(operation.ExpectedVersion);
var resealed = PayloadReseal.TryReseal(
keyring,
vaultId,
operation.EntityType,
operation.EntityId,
payload,
openAtVersion: version,
sealAtVersion: version);
if (resealed is null)
{
// The generation this change was queued under is one this session no longer holds. It goes
// as it stands: the server takes it either way, and a queued edit that cannot be re-sealed
// is still the user's work.
return payload;
}
await outbox.ReviseAsync(
operation.Sequence,
operation.Operation,
operation.ExpectedVersion,
resealed,
operation.Fields,
operation.Ancestor,
cancellationToken)
.ConfigureAwait(false);
return resealed;
}
/// Whether this answer warrants another push round.
private async Task HandleAsync(
Guid vaultId,
PendingOperation operation,
SyncPushResult result,
SyncReportBuilder report,
CancellationToken cancellationToken)
{
switch (result.Status)
{
case SyncOperationStatus.Applied:
case SyncOperationStatus.Duplicate:
// Duplicate means an earlier push of this exact operation id already landed, so the
// stored state is what this operation intended. Treated as success on purpose: that is
// what makes a retry after a timeout exactly-once rather than merely at-least-once.
await AcceptAsync(vaultId, operation, result, cancellationToken).ConfigureAwait(false);
report.Pushed++;
// Counted separately so the interface can stay quiet about a pass that carried nothing but
// log entries. Every user action now queues one, so without this the background pass would
// have something to announce after literally every save — and would overwrite the message
// the save itself had just put on the status line.
if (IsLog(operation.EntityType))
{
report.PushedLogEntries++;
}
return false;
case SyncOperationStatus.Conflict:
return await ResolveAsync(vaultId, operation, result, report, cancellationToken)
.ConfigureAwait(false);
case SyncOperationStatus.Forbidden:
await RejectAsync(
vaultId,
operation,
"You no longer have permission to change this item.",
report,
cancellationToken).ConfigureAwait(false);
return false;
case SyncOperationStatus.Invalid:
await RejectAsync(
vaultId,
operation,
result.Detail ?? "The server rejected this change as invalid.",
report,
cancellationToken).ConfigureAwait(false);
return false;
default:
await outbox.RecordFailureAsync(
operation.Sequence,
$"Unexpected push status {result.Status}.",
cancellationToken).ConfigureAwait(false);
return false;
}
}
/// Records an accepted operation and clears it from the outbox.
private async Task AcceptAsync(
Guid vaultId,
PendingOperation operation,
SyncPushResult result,
CancellationToken cancellationToken)
{
var expected = SyncVersions.NextVersion(operation.ExpectedVersion);
var version = result.Version ?? expected;
var isDelete = operation.Operation == SyncOperation.Delete;
// The payload was sealed at the version this client predicted, and the AAD binds that version.
// If the server assigned a different one — which its own version check should make impossible —
// storing the payload would leave a mirror row that never decrypts. Skip the write; the pull at
// the end of the pass brings the authoritative row.
var canMirror = isDelete || version == expected;
if (canMirror)
{
await items.SaveAsync(
new StoredItem(
vaultId,
operation.EntityType,
operation.EntityId,
version,
result.ChangeSequence ?? 0,
isDelete ? null : operation.Payload,
isDelete ? null : operation.Fields,
isDelete,
clock.GetUtcNow()),
cancellationToken).ConfigureAwait(false);
}
await outbox.CompleteAsync(operation.Sequence, cancellationToken).ConfigureAwait(false);
}
private async Task ResolveAsync(
Guid vaultId,
PendingOperation operation,
SyncPushResult result,
SyncReportBuilder report,
CancellationToken cancellationToken)
{
if (operation.Attempts >= options.MaxAttemptsBeforeParking)
{
await RejectAsync(
vaultId,
operation,
string.Create(
CultureInfo.InvariantCulture,
$"Could not be reconciled after {operation.Attempts} attempts."),
report,
cancellationToken).ConfigureAwait(false);
return false;
}
if (!reconcilers.TryGetValue(operation.EntityType, out var reconciler))
{
// Only reachable if something queued a type this build does not synchronise, which the
// repositories cannot do. Parked rather than dropped, so the change is visible to a user
// instead of retried for ever against a path that cannot handle it.
await RejectAsync(
vaultId,
operation,
$"This version of DodoSSH cannot reconcile items of type {operation.EntityType}.",
report,
cancellationToken).ConfigureAwait(false);
return false;
}
if (result.ServerEntity is null)
{
// The version check failed but the server has no such row. Re-offer it as a create.
return await RetryAsCreateAsync(vaultId, reconciler, operation, report, cancellationToken)
.ConfigureAwait(false);
}
await ItemMirror.WriteAsync(items, vaultId, result.ServerEntity, cancellationToken)
.ConfigureAwait(false);
await reconciler
.ReconcileAsync(vaultId, result.ServerEntity, operation, report, cancellationToken)
.ConfigureAwait(false);
return true;
}
private async Task RetryAsCreateAsync(
Guid vaultId,
IItemReconciler reconciler,
PendingOperation operation,
SyncReportBuilder report,
CancellationToken cancellationToken)
{
if (operation.Operation == SyncOperation.Delete)
{
// Nothing there to delete, so the intent is already satisfied.
await outbox.CompleteAsync(operation.Sequence, cancellationToken).ConfigureAwait(false);
return false;
}
// Re-sealing needs the item's own cipher, so the reconciler does it. A reason back means the
// change can never be sent, not that it should be retried.
var failure = await reconciler
.ReofferAsCreateAsync(vaultId, operation, cancellationToken)
.ConfigureAwait(false);
if (failure is null)
{
return true;
}
await RejectAsync(vaultId, operation, failure, report, cancellationToken).ConfigureAwait(false);
return false;
}
/// Parks an operation the server will never accept, and says why.
private async Task RejectAsync(
Guid vaultId,
PendingOperation operation,
string reason,
SyncReportBuilder report,
CancellationToken cancellationToken)
{
await outbox.ParkAsync(operation.Sequence, reason, cancellationToken).ConfigureAwait(false);
await conflicts.RecordAsync(
vaultId,
operation.EntityType,
operation.EntityId,
ConflictKind.Rejected,
ConflictDetails.Encode(reason),
cancellationToken).ConfigureAwait(false);
report.Parked++;
}
}