DodoSSH/tests/DodoSSH.Client.Sync.Tests/SyncEngineTests.cs

using DodoSSH.Client.Api;
using DodoSSH.Contracts;
using static DodoSSH.Client.Sync.Tests.SyncHarness;

namespace DodoSSH.Client.Sync.Tests;

/// <summary>
/// The mechanics of a pass: paging, batching, bounds, and what the cursor is allowed to be.
/// </summary>
/// <remarks>
/// Separate from the conflict matrix because the failure modes are different. Here a mistake shows up as
/// a sync that never finishes, or one that quietly stops halfway and reports success.
/// </remarks>
public sealed class SyncEngineTests
{
    [Fact]
    public async Task APullLargerThanOnePage_ReadsEveryChange()
    {
        // The server clamps a client's requested limit, so a client that trusted one response to be the
        // whole story would silently see part of a vault.
        using var harness = await CreateAsync(
            new SyncOptions { PullPageSize = 2, MaxOperationsPerPush = 100 });

        harness.Server.MaxPullLimit = 2;

        for (var index = 0; index < 7; index++)
        {
            await harness.First.CreateAsync(Host($"host-{index}"));
        }

        await harness.First.SyncAsync();

        var report = await harness.Second.SyncAsync();

        report.Pulled.ShouldBe(7);
        (await harness.Second.ListAsync()).Items.Count.ShouldBe(7);
    }

    [Fact]
    public async Task MoreQueuedChangesThanOneBatch_AreAllPushed()
    {
        using var harness = await CreateAsync(new SyncOptions { MaxOperationsPerPush = 2 });

        for (var index = 0; index < 5; index++)
        {
            await harness.First.CreateAsync(Host($"host-{index}"));
        }

        var report = await harness.First.SyncAsync();

        report.Pushed.ShouldBe(5);
        harness.Server.RowCount.ShouldBe(5);

        // Three rounds of two, so the drain loop genuinely continued rather than stopping at one batch.
        harness.Server.PushCount.ShouldBeGreaterThanOrEqualTo(3);
    }

    [Fact]
    public async Task AnExhaustedPushLoop_SaysSoRatherThanPretendingItFinished()
    {
        // A bound is necessary — each round advances, but against a vault someone else writes to
        // continuously a pass could keep finding work. Reporting it is what stops that looking like
        // success.
        using var harness = await CreateAsync(
            new SyncOptions { MaxOperationsPerPush = 1, MaxPushRounds = 2 });

        for (var index = 0; index < 5; index++)
        {
            await harness.First.CreateAsync(Host($"host-{index}"));
        }

        var report = await harness.First.SyncAsync();

        report.RoundsExhausted.ShouldBeTrue();
        report.Pushed.ShouldBe(2);

        // And the rest is still queued, not lost.
        (await harness.First.Outbox.TakeAsync(VaultId, 100, TestContext.Current.CancellationToken))
            .Count.ShouldBe(3);

        // A further pass picks up where this one stopped.
        await harness.First.SyncAsync();
        await harness.First.SyncAsync();
        harness.Server.RowCount.ShouldBe(5);
    }

    [Fact]
    public async Task TheCursor_IsWhateverTheServerIssued()
    {
        // Opaque and integrity-tagged. The fake server rejects a cursor it did not mint, so a client that
        // computed one would fail here rather than quietly resuming from a position it invented.
        using var harness = await CreateAsync();

        await harness.First.CreateAsync(Host("prod-db"));
        await harness.First.SyncAsync();

        var state = await harness.First.SyncState.ReadAsync(
            VaultId, TestContext.Current.CancellationToken);

        state.Cursor.ShouldNotBeNull();
        state.Cursor.ShouldStartWith("fake-v1:");
    }

    [Fact]
    public async Task AnEmptyPull_DoesNotMoveTheCursor()
    {
        // If it did, a write landing between this read and the next would be skipped for ever.
        using var harness = await CreateAsync();

        await harness.First.SyncAsync();

        var before = await harness.First.SyncState.ReadAsync(
            VaultId, TestContext.Current.CancellationToken);

        await harness.First.SyncAsync();

        var after = await harness.First.SyncState.ReadAsync(
            VaultId, TestContext.Current.CancellationToken);

        after.Cursor.ShouldBe(before.Cursor);
    }

    [Fact]
    public async Task ARefusedCursor_ReadsTheVaultAgainFromTheBeginning()
    {
        // The server's cursor signing key was rotated, or the vault is being served from a restored
        // database. The stored position is refused for good, so a pass that only reported the 400 would
        // leave this machine frozen at it — asking once a minute, for ever, and being told the same thing.
        using var harness = await CreateAsync();

        await harness.First.SyncAsync();

        var entityId = await harness.Second.CreateAsync(Host("prod-db"));
        await harness.Second.SyncAsync();

        harness.Server.RefuseCursors = true;

        var report = await harness.First.SyncAsync();

        report.ResyncedFromStart.ShouldBeTrue();
        report.Pulled.ShouldBe(1);

        // And the point of all of it: the change that was on the far side of the refused position is here.
        (await harness.First.FindAsync(entityId)).Secret.Label.ShouldBe("prod-db");
    }

    [Fact]
    public async Task ARefusedCursor_DoesNotStrandWhatIsWaitingToBePushed()
    {
        // The half that made this worth recovering from rather than merely reporting. The pull runs first,
        // so a pass that gave up on the refusal never reached the outbox at all: every edit made on this
        // machine stayed queued behind a position the server was never going to accept again.
        using var harness = await CreateAsync();

        await harness.First.SyncAsync();

        var entityId = await harness.First.CreateAsync(Host("prod-db"));

        harness.Server.RefuseCursors = true;

        var report = await harness.First.SyncAsync();

        report.ResyncedFromStart.ShouldBeTrue();
        report.Pushed.ShouldBe(1);
        harness.Server.Find(entityId).ShouldNotBeNull();
    }

    [Fact]
    public async Task ARefusalOfTheBeginningItself_IsReportedRatherThanRetried()
    {
        // "From the beginning" is the one position a client is allowed to ask for, so a server that
        // refuses it is one this code cannot reason about. Saying so beats replaying the log against it
        // until a page bound runs out.
        using var harness = await CreateAsync();

        await harness.First.SyncAsync();

        harness.Server.RefuseCursors = true;
        harness.Server.RefuseEvenTheBeginning = true;

        var failure = await Should.ThrowAsync<DodoSshApiException>(
            async () => await harness.First.SyncAsync());

        failure.Code.ShouldBe(ProblemCodes.InvalidCursor);
    }

    [Fact]
    public async Task ClockSkew_IsRecordedAndNotActedOn()
    {
        // Recorded because a user should be able to see it. Not acted on because the merge decides by
        // version and retained ancestor — a skewed clock must not be able to pick a winner.
        using var harness = await CreateAsync();

        harness.Server.Now = TimeProvider.System.GetUtcNow().AddHours(3);

        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        var report = await harness.First.SyncAsync();

        report.ServerTimeSkewMs.ShouldBeGreaterThan(2 * 60 * 60 * 1000);

        // The item still round-trips, so nothing downstream depended on the timestamp.
        (await harness.First.FindAsync(entityId)).Secret.Label.ShouldBe("prod-db");
    }

    [Fact]
    public async Task ASyncWithNothingToDo_TouchesTheServerOnceAndReportsNothing()
    {
        using var harness = await CreateAsync();

        var report = await harness.First.SyncAsync();

        report.Pulled.ShouldBe(0);
        report.Pushed.ShouldBe(0);
        report.NeedsAttention.ShouldBeFalse();
        harness.Server.PushCount.ShouldBe(0);
    }

    [Fact]
    public async Task AVaultWithNoUsableGrant_IsReportedRatherThanRead()
    {
        // A grant awaiting re-wrap after a rekey. The vault is temporarily unreadable and saying so is
        // the only honest answer — showing an empty host list would be indistinguishable from an empty
        // vault.
        using var harness = await CreateAsync();

        var unknown = Guid.CreateVersion7();

        harness.First.Keyring.CanRead(unknown).ShouldBeFalse();

        await Should.ThrowAsync<VaultUnreadableException>(
            async () => await harness.First.Hosts.ListAsync(
                unknown, TestContext.Current.CancellationToken));
    }

    /// <summary>
    /// A save made while the previous save of the same host is on its way to the server survives that push
    /// being acknowledged, and reaches the server as the newer state rather than as a conflict.
    /// </summary>
    /// <remarks>
    /// The outbox coalesces the second save into the row the push read. Acknowledging by sequence alone
    /// deleted that row — the newer state with it — and the server and both machines kept the older one.
    /// </remarks>
    [Fact]
    public async Task ASaveMadeWhileAPushIsInFlight_IsNotLostWhenThatPushIsAcknowledged()
    {
        using var harness = await CreateAsync();

        var id = await harness.First.CreateAsync(Host("prod-db", hostname: "first.internal"));
        await harness.First.SyncAsync();

        await harness.First.UpdateAsync(id, Host("prod-db", hostname: "second.internal"));

        harness.Server.OnPush = () => harness.First
            .UpdateAsync(id, Host("prod-db", hostname: "third.internal"))
            .GetAwaiter()
            .GetResult();

        await harness.First.SyncAsync();
        await harness.First.SyncAsync();

        (await harness.First.FindAsync(id)).Secret.Hostname.ShouldBe("third.internal");
        (await harness.First.ConflictsAsync()).ShouldBeEmpty();

        await harness.Second.SyncAsync();

        (await harness.Second.FindAsync(id)).Secret.Hostname.ShouldBe("third.internal");
    }

    /// <summary>
    /// A cursor read under a shorter list of item types starts the pull over, once.
    /// </summary>
    /// <remarks>
    /// The cursor a pull returns is past every change it read, including the changes of types it did not ask
    /// for. A build that adds a type to the list — ObjectStore did — and carried on from
    /// that cursor would never receive the items of the new type made before it arrived.
    /// </remarks>
    [Fact]
    public async Task ACursorReadUnderAnotherListOfTypes_StartsThePullOverOnce()
    {
        using var harness = await CreateAsync();

        await harness.First.CreateAsync(Host("prod-db"));
        await harness.First.SyncAsync();

        var state = await harness.First.SyncState.ReadAsync(VaultId, TestContext.Current.CancellationToken);
        state.Cursor.ShouldNotBeNull();
        state.PulledTypes.ShouldNotBeNull();

        // What an older build, which pulled fewer types, would have left behind.
        await harness.First.SyncState.SaveAsync(
            state with { PulledTypes = "1" }, TestContext.Current.CancellationToken);

        (await harness.First.SyncAsync()).ResyncedFromStart.ShouldBeTrue();
        (await harness.First.SyncAsync()).ResyncedFromStart.ShouldBeFalse("the list is recorded with the cursor");

        (await harness.First.ListAsync()).Items.ShouldHaveSingleItem();
    }
}