Sync: after a read from the beginning, remove what the server no longer has, and flag an edit that brings a forgotten item back

The server now forgets a deletion past its retention and refuses cursors
from before it. The client then reads everything again, but everything
again says nothing about what went. So a machine away for longer than the
retention kept those items for good, and its next edit of one put it back
on the server for everyone.

A read from the beginning now marks each item it reads, keeping the mark in
the sync state because the read can span passes. When it finishes, it
removes the items of the synced types it never marked. Two kinds are kept:
items newer than the read's bound, which this machine pushed during it, and
items with a change still waiting, which are the user's work. For those,
the push finds the server without the item and sends it as a create, as
before, but now records a RecreatedAfterCollection conflict and counts it,
so the sync message says so instead of undoing someone's deletion silently.

The bound is the highest sequence the cache held when the read began, or
anything higher it has read since. Taking only what the read returned would
have left the last item of a vault in place for ever.

tests/DodoSSH.Client.Sync.Tests/FakeVaultServer.cs

@@ -102,6 +102,7 @@ internal sealed class FakeVaultServer : ISyncApi
         SyncPullRequest request,
         CancellationToken cancellationToken)
     {
+        var issuedUnder = horizon;
         var after = DecodeCursor(request.Cursor);
         var limit = Math.Clamp(request.Limit ?? MaxPullLimit, 1, MaxPullLimit);
 
@@ -128,7 +129,7 @@ internal sealed class FakeVaultServer : ISyncApi
 
         return Task.FromResult(new SyncPullResponse(
             [.. page.Select(entry => Hydrate(entry))],
-            EncodeCursor(next),
+            EncodeCursor(next, issuedUnder),
             hasMore,
             Now,
             KeyGeneration));
@@ -206,7 +207,25 @@ internal sealed class FakeVaultServer : ISyncApi
     internal Row? Find(Guid entityId, SyncEntityType entityType = SyncEntityType.Host) =>
         rows.TryGetValue((entityType, entityId), out var row) ? row : null;
 
-    private long Head => log.Count == 0 ? 0 : log[^1].Sequence;
+    /// <summary>
+    /// Forgets every deletion, as the real server does once one is past its retention: the tombstone and
+    /// its log entries go, and a cursor from before them is refused from then on.
+    /// </summary>
+    internal void CollectTombstones()
+    {
+        foreach (var (key, row) in rows.Where(entry => entry.Value.IsDeleted).ToList())
+        {
+            rows.Remove(key);
+            log.RemoveAll(entry => entry.EntityType == key.Type && entry.EntityId == key.EntityId);
+            horizon = Math.Max(horizon, row.ChangeSequence);
+        }
+    }
+
+    /// <summary>The newest sequence ever issued, which a collection must not take back.</summary>
+    private long Head => Math.Max(log.Count == 0 ? 0 : log[^1].Sequence, horizon);
+
+    /// <summary>The last deletion forgotten; see <see cref="CollectTombstones"/>.</summary>
+    private long horizon;
 
     // ---- The decision table ----
 
@@ -481,8 +500,12 @@ internal sealed class FakeVaultServer : ISyncApi
     /// rejects. The real server HMAC-tags them; the property that matters to the client is only that it
     /// must round-trip what it is given.
     /// </remarks>
-    private static string EncodeCursor(long sequence) =>
-        "fake-v1:" + sequence.ToString(CultureInfo.InvariantCulture);
+    /// <remarks>
+    /// Carries the horizon it was read under, as the real one does, so a read from the beginning that started
+    /// after a collection is not refused on its own early pages. See <c>DodoSSH.Domain.Sync.SyncCursor</c>.
+    /// </remarks>
+    private static string EncodeCursor(long sequence, long issuedUnder = 0) =>
+        string.Create(CultureInfo.InvariantCulture, $"fake-v1:{sequence}@{issuedUnder}");
 
     private long DecodeCursor(string? cursor)
     {
@@ -502,12 +525,26 @@ internal sealed class FakeVaultServer : ISyncApi
             return 0;
         }
 
+        var at = cursor.IndexOf('@', StringComparison.Ordinal);
+
         if (!cursor.StartsWith("fake-v1:", StringComparison.Ordinal)
-            || !long.TryParse(cursor.AsSpan(8), CultureInfo.InvariantCulture, out var sequence))
+            || at < 0
+            || !long.TryParse(cursor.AsSpan(8, at - 8), CultureInfo.InvariantCulture, out var sequence)
+            || !long.TryParse(cursor.AsSpan(at + 1), CultureInfo.InvariantCulture, out var issuedUnder))
         {
             throw new InvalidOperationException($"A client sent a cursor it should not have: '{cursor}'.");
         }
 
+        if (sequence < horizon && issuedUnder < horizon)
+        {
+            // From before a deletion this server no longer remembers, so it could never be told of it.
+            throw new DodoSshApiException(
+                HttpStatusCode.BadRequest,
+                ProblemCodes.InvalidCursor,
+                "The server returned 400: The sync cursor is not valid for this vault. "
+                + "Resync from the beginning.");
+        }
+
         return sequence;
     }
 

tests/DodoSSH.Client.Sync.Tests/TombstoneCollectionTests.cs

@@ -0,0 +1,142 @@
+using DodoSSH.Client.Storage;
+using static DodoSSH.Client.Sync.Tests.SyncHarness;
+
+namespace DodoSSH.Client.Sync.Tests;
+
+/// <summary>
+/// A machine away for longer than the server keeps deletions (ADR 0003).
+/// </summary>
+/// <remarks>
+/// The server forgets a deletion once it is past its retention and refuses any cursor from before it. The
+/// machine then reads everything again — which says nothing about what is gone — so what it held and did
+/// not read again is what has to go. Except the user's own unsent work, which is kept and said out loud.
+/// </remarks>
+public sealed class TombstoneCollectionTests
+{
+    [Fact]
+    public async Task AnItemDeletedAndForgottenWhileAway_IsRemovedByTheReadFromTheBeginning()
+    {
+        using var harness = await CreateAsync();
+
+        var gone = await harness.Second.CreateAsync(Host("old-db"));
+        var kept = await harness.Second.CreateAsync(Host("prod-db"));
+        await harness.Second.SyncAsync();
+        await harness.First.SyncAsync();
+
+        await harness.Second.DeleteAsync(gone);
+        await harness.Second.SyncAsync();
+        harness.Server.CollectTombstones();
+
+        var report = await harness.First.SyncAsync();
+
+        report.ResyncedFromStart.ShouldBeTrue();
+        report.Pruned.ShouldBe(1);
+        (await harness.First.ListAsync()).Items.Select(item => item.EntityId).ShouldBe([kept]);
+    }
+
+    [Fact]
+    public async Task TheOnlyItemInAVault_IsRemovedAsWell()
+    {
+        // The read from the beginning brings back nothing at all, so a sweep bounded by what it read would
+        // reach nothing either. It is bounded by what the cache held when it started.
+        using var harness = await CreateAsync();
+
+        var gone = await harness.Second.CreateAsync(Host("old-db"));
+        await harness.Second.SyncAsync();
+        await harness.First.SyncAsync();
+
+        await harness.Second.DeleteAsync(gone);
+        await harness.Second.SyncAsync();
+        harness.Server.CollectTombstones();
+
+        var report = await harness.First.SyncAsync();
+
+        report.Pruned.ShouldBe(1);
+        (await harness.First.ListAsync()).Items.ShouldBeEmpty();
+    }
+
+    [Fact]
+    public async Task AnItemWithAnUnsentEdit_IsKept_PutBack_AndSaidSo()
+    {
+        using var harness = await CreateAsync();
+
+        var entityId = await harness.Second.CreateAsync(Host("old-db"));
+        await harness.Second.SyncAsync();
+        await harness.First.SyncAsync();
+
+        await harness.First.UpdateAsync(entityId, Host("old-db, still needed"));
+
+        await harness.Second.DeleteAsync(entityId);
+        await harness.Second.SyncAsync();
+        harness.Server.CollectTombstones();
+
+        var report = await harness.First.SyncAsync();
+
+        report.Pruned.ShouldBe(0);
+        report.Recreated.ShouldBe(1);
+        report.NeedsAttention.ShouldBeTrue();
+
+        (await harness.First.FindAsync(entityId)).Secret.Label.ShouldBe("old-db, still needed");
+        harness.Server.Find(entityId).ShouldNotBeNull().IsDeleted.ShouldBeFalse();
+
+        (await harness.First.ConflictsAsync())
+            .ShouldContain(conflict => conflict.EntityId == entityId
+                && conflict.Kind == ConflictKind.RecreatedAfterCollection);
+
+        await harness.Second.SyncAsync();
+        (await harness.Second.FindAsync(entityId)).Secret.Label.ShouldBe("old-db, still needed");
+    }
+
+    [Fact]
+    public async Task AReadSpreadOverSeveralPasses_SweepsAtTheEnd_AndKeepsWhatWasPushedMeanwhile()
+    {
+        // Pages per pass are capped, so the read from the beginning can take more than one pass, and what it
+        // has seen has to outlast each one. A host made here during it is pushed between the passes.
+        using var harness = await CreateAsync(new SyncOptions { PullPageSize = 1, MaxPullPages = 2 });
+
+        harness.Server.MaxPullLimit = 1;
+
+        var gone = await harness.Second.CreateAsync(Host("old-db"));
+        var kept = new List<Guid>();
+
+        for (var index = 0; index < 4; index++)
+        {
+            kept.Add(await harness.Second.CreateAsync(Host($"host-{index}")));
+        }
+
+        // Settled by hand: one pass reads at most two pages, and each needs several.
+        for (var pass = 0; pass < 6; pass++)
+        {
+            await harness.Second.SyncAsync();
+            await harness.First.SyncAsync();
+        }
+
+        await harness.Second.DeleteAsync(gone);
+        await harness.Second.SyncAsync();
+        harness.Server.CollectTombstones();
+
+        kept.Add(await harness.First.CreateAsync(Host("made-meanwhile")));
+
+        var pruned = 0;
+
+        for (var pass = 0; pass < 10; pass++)
+        {
+            pruned += (await harness.First.SyncAsync()).Pruned;
+
+            var state = await harness.First.SyncState.ReadAsync(VaultId, TestContext.Current.CancellationToken);
+
+            if (state.SweepMark is null)
+            {
+                break;
+            }
+
+            // Mid-sweep, nothing has gone yet.
+            pruned.ShouldBe(0);
+            (await harness.First.ListAsync()).Items.Count.ShouldBe(6);
+        }
+
+        pruned.ShouldBe(1);
+        (await harness.First.ListAsync()).Items.Select(item => item.EntityId).ShouldBe(kept, ignoreOrder: true);
+        harness.Server.Find(kept[^1]).ShouldNotBeNull();
+    }
+}