DodoSSH/tests/DodoSSH.Client.Sync.Tests/ConflictMatrixTests.cs

using DodoSSH.Client.Storage;
using static DodoSSH.Client.Sync.Tests.SyncHarness;

namespace DodoSSH.Client.Sync.Tests;

/// <summary>
/// Two machines, one vault, every way they can disagree.
/// </summary>
/// <remarks>
/// The highest-value suite in the product, because this is the only place data can be lost. Every case
/// asserts two things: that the two devices converge on the same state, and that whatever the merge had
/// to override is recorded rather than gone. A merge that quietly drops the password someone just typed
/// is worse than one that refuses to merge at all.
/// </remarks>
public sealed class ConflictMatrixTests : IAsyncLifetime
{
    private SyncHarness harness = null!;

    /// <inheritdoc />
    public async ValueTask InitializeAsync() => harness = await CreateAsync();

    /// <inheritdoc />
    public ValueTask DisposeAsync()
    {
        harness.Dispose();
        return ValueTask.CompletedTask;
    }

    // ---- The uncontested paths ----

    [Fact]
    public async Task ACreatedHost_ReachesTheOtherMachine()
    {
        var entityId = await harness.First.CreateAsync(Host("prod-db", notes: "primary"));

        await harness.SettleAsync();

        var seen = await harness.Second.FindAsync(entityId);
        seen.Secret.Label.ShouldBe("prod-db");
        seen.Secret.Notes.ShouldBe("primary");
        seen.HasUnsyncedChanges.ShouldBeFalse();
        harness.Server.RowCount.ShouldBe(1);
    }

    [Fact]
    public async Task AHostCreatedOffline_IsVisibleLocallyBeforeAnySync()
    {
        // The reason the outbox exists. A host typed in on a plane has to be usable on that plane.
        var entityId = await harness.First.CreateAsync(Host("prod-db"));

        var local = await harness.First.FindAsync(entityId);
        local.Secret.Label.ShouldBe("prod-db");
        local.HasUnsyncedChanges.ShouldBeTrue();

        harness.Server.RowCount.ShouldBe(0);
    }

    [Fact]
    public async Task ALocalOnlyEdit_IsPushed()
    {
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        await harness.First.UpdateAsync(entityId, Host("prod-db", notes: "rotate quarterly"));
        await harness.SettleAsync();

        (await harness.Second.FindAsync(entityId)).Secret.Notes.ShouldBe("rotate quarterly");
        (await harness.First.ConflictsAsync()).ShouldBeEmpty();
    }

    [Fact]
    public async Task ARemoteOnlyEdit_IsPulledWithoutAConflict()
    {
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        await harness.Second.UpdateAsync(entityId, Host("prod-db", username: "postgres"));
        await harness.SettleAsync();

        (await harness.First.FindAsync(entityId)).Secret.Username.ShouldBe("postgres");
        (await harness.First.ConflictsAsync()).ShouldBeEmpty();
    }

    // ---- Both edited ----

    [Fact]
    public async Task BothEditedDifferentFields_BothSurvive()
    {
        // The payoff for a field-level merge. Last-writer-wins would lose one of these.
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        await harness.First.UpdateAsync(entityId, Host("prod-db", notes: "from the laptop"));
        await harness.Second.UpdateAsync(entityId, Host("prod-db", username: "postgres"));

        await harness.SettleAsync();

        var merged = (await harness.First.FindAsync(entityId)).Secret;
        merged.Notes.ShouldBe("from the laptop");
        merged.Username.ShouldBe("postgres");

        (await harness.Second.FindAsync(entityId)).Secret.ShouldBe(merged);
        (await harness.First.ConflictsAsync()).ShouldBeEmpty();
    }

    [Fact]
    public async Task BothAddedADifferentDirective_BothSurvive()
    {
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        await harness.First.UpdateAsync(
            entityId, Host("prod-db", options: [("Compression", "yes")]));
        await harness.Second.UpdateAsync(
            entityId, Host("prod-db", options: [("ServerAliveInterval", "30")]));

        await harness.SettleAsync();

        var merged = (await harness.First.FindAsync(entityId)).Secret;
        merged.Options.Count.ShouldBe(2);
        merged.Options.TryGetValue("Compression", out _).ShouldBeTrue();
        merged.Options.TryGetValue("ServerAliveInterval", out _).ShouldBeTrue();
    }

    [Fact]
    public async Task BothEditedTheSameField_OneValueWinsAndTheOtherIsRecorded()
    {
        // A genuine clash. Whichever side loses, its value has to be retrievable — that is the entire
        // justification for resolving automatically instead of blocking.
        var entityId = await harness.First.CreateAsync(Host("prod-db", notes: "original"));
        await harness.SettleAsync();

        await harness.First.UpdateAsync(entityId, Host("prod-db", notes: "from the laptop"));
        await harness.Second.UpdateAsync(entityId, Host("prod-db", notes: "from the desktop"));

        await harness.SettleAsync();

        var first = (await harness.First.FindAsync(entityId)).Secret;
        var second = (await harness.Second.FindAsync(entityId)).Secret;

        first.ShouldBe(second);

        var winner = first.Notes.ShouldNotBeNull();
        var lost = string.Equals(winner, "from the laptop", StringComparison.Ordinal)
            ? "from the desktop"
            : "from the laptop";

        // One of the two, and the same one on both machines. Which is not the point; that the other is
        // retrievable is.
        new[] { "from the laptop", "from the desktop" }
            .Contains(winner, StringComparer.Ordinal)
            .ShouldBeTrue();

        (await ConflictsAcrossDevicesAsync())
            .ShouldContain(kind => kind == ConflictKind.FieldOverridden);

        (await DiscardedValuesAsync())
            .ShouldContain(
                detail => detail.Contains(lost, StringComparison.Ordinal),
                "the overridden value must be recoverable from the conflict log");
    }

    [Fact]
    public async Task BothMadeTheSameEdit_IsNotAConflict()
    {
        // Two people fixing the same typo must not be asked to arbitrate.
        var entityId = await harness.First.CreateAsync(Host("prod-db", hostname: "db.internl"));
        await harness.SettleAsync();

        await harness.First.UpdateAsync(entityId, Host("prod-db", hostname: "db.internal"));
        await harness.Second.UpdateAsync(entityId, Host("prod-db", hostname: "db.internal"));

        await harness.SettleAsync();

        (await harness.First.FindAsync(entityId)).Secret.Hostname.ShouldBe("db.internal");
        (await ConflictsAcrossDevicesAsync()).ShouldBeEmpty();
    }

    // ---- Deletes ----

    [Fact]
    public async Task ADeletedHost_DisappearsEverywhere()
    {
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        await harness.First.DeleteAsync(entityId);
        await harness.SettleAsync();

        (await harness.First.ListAsync()).Items.ShouldBeEmpty();
        (await harness.Second.ListAsync()).Items.ShouldBeEmpty();
        harness.Server.RowCount.ShouldBe(0);
    }

    [Fact]
    public async Task BothDeleted_IsNotAConflict()
    {
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        await harness.First.DeleteAsync(entityId);
        await harness.Second.DeleteAsync(entityId);

        await harness.SettleAsync();

        (await harness.First.ListAsync()).Items.ShouldBeEmpty();
        (await harness.Second.ListAsync()).Items.ShouldBeEmpty();
        (await ConflictsAcrossDevicesAsync()).ShouldBeEmpty();
    }

    [Fact]
    public async Task DeletedElsewhereWhileEditedHere_TheLocalWorkSurvivesUnderANewName()
    {
        // The case where naive handling loses data outright. The tombstone has to stand — arguing with it
        // conflicts for ever — so the edit is preserved as a separate host instead of being dropped.
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        // The laptop syncs first, so its delete is what reaches the server; the desktop's edit is the
        // one that has to be rescued. Which side loses is decided by who gets there first, and both
        // orderings are covered — see EditedElsewhereWhileDeletedHere for the mirror image.
        await harness.First.DeleteAsync(entityId);
        await harness.Second.UpdateAsync(
            entityId, Host("prod-db", notes: "credentials rotated, do not delete"));

        await harness.SettleAsync();

        var listing = await harness.First.ListAsync();

        var restored = listing.Items.ShouldHaveSingleItem();
        restored.EntityId.ShouldNotBe(entityId);
        restored.Secret.Label.ShouldBe("prod-db (restored)");
        restored.Secret.Notes.ShouldBe("credentials rotated, do not delete");

        (await ConflictsAcrossDevicesAsync())
            .ShouldContain(kind => kind == ConflictKind.RemoteDeleteResurrected);

        // And the other machine sees it too, so the rescue is not local-only.
        (await harness.Second.ListAsync()).Items.ShouldHaveSingleItem()
            .EntityId.ShouldBe(restored.EntityId);
    }

    [Fact]
    public async Task ReplayingAPulledDeletion_DoesNotDuplicateTheRescuedCopy()
    {
        // Applying a pulled change is at-least-once — the cursor is saved after the page is applied — so
        // a whole page can arrive twice. This checks the replay is harmless end to end; the deterministic
        // id it relies on is pinned by ResurrectionIdTests.
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        await harness.First.DeleteAsync(entityId);
        await harness.Second.UpdateAsync(entityId, Host("prod-db", notes: "keep me"));
        await harness.First.SyncAsync();

        // Rewind the desktop's cursor, so the deletion arrives a second time and it tries to rescue the
        // same content twice.
        await harness.Second.SyncAsync();
        await harness.Second.SyncState.ResetAsync(VaultId, TestContext.Current.CancellationToken);
        await harness.Second.SyncAsync();

        await harness.SettleAsync();

        (await harness.First.ListAsync()).Items.Count.ShouldBe(1);
        harness.Server.RowCount.ShouldBe(1);
    }

    [Fact]
    public async Task EditedElsewhereWhileDeletedHere_TheDeleteIsAbandonedAndReported()
    {
        // The mirror image, and resolved the same way round: an edit outlives a removal. Re-deleting
        // costs a click; a discarded edit may be the only copy of something.
        var entityId = await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        await harness.First.UpdateAsync(entityId, Host("prod-db", notes: "still needed"));
        await harness.Second.DeleteAsync(entityId);

        await harness.SettleAsync();

        var survivor = await harness.First.FindAsync(entityId);
        survivor.Secret.Notes.ShouldBe("still needed");

        (await ConflictsAcrossDevicesAsync())
            .ShouldContain(kind => kind == ConflictKind.LocalDeleteOverridden);
    }

    // ---- Ordering, retries and idempotence ----

    [Fact]
    public async Task OfflineChanges_ArePushedInTheOrderTheyWereMade()
    {
        var first = await harness.First.CreateAsync(Host("a-bastion"));
        var second = await harness.First.CreateAsync(Host("b-database"));
        var third = await harness.First.CreateAsync(Host("c-cache"));

        await harness.SettleAsync();

        var order = (await harness.Second.ListAsync()).Items
            .OrderBy(host => host.Version)
            .ThenBy(host => host.Secret.Label, StringComparer.Ordinal)
            .Select(host => host.EntityId)
            .ToArray();

        order.ShouldBe([first, second, third], ignoreOrder: false);
    }

    [Fact]
    public async Task ASecondSyncPass_ChangesNothing()
    {
        // Idempotence, which is what makes re-reading a client's own writes a safe way to avoid the
        // cursor-gap hazard in the push response.
        await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        var before = await harness.First.HostsSortedAsync();
        var pushesBefore = harness.Server.PushCount;

        var report = await harness.First.SyncAsync();

        report.Pushed.ShouldBe(0);
        harness.Server.PushCount.ShouldBe(pushesBefore);
        (await harness.First.HostsSortedAsync()).ShouldBe(before);
    }

    [Fact]
    public async Task AnEditWhileAnEarlierPushIsUnacknowledged_KeepsTheNewerValueWithoutDuplicating()
    {
        // A create that reached the server and whose answer did not come back, followed by another edit.
        // The newer value has to win and there must be exactly one host afterwards. Two mechanisms keep
        // that true: the pull sees the server's row and re-bases the queued edit onto it, and a coalesced
        // row carries a fresh operation id so the server cannot answer Duplicate — "already applied" —
        // for an operation whose contents have since changed. The second is pinned directly by
        // OutboxStoreTests.ACoalescedEdit_GetsAFreshOperationId; here they are exercised together.
        var entityId = await harness.First.CreateAsync(Host("prod-db", notes: "first"));

        // The create lands on the server, but the acknowledgement never reaches the laptop.
        await PushBehindTheEnginesBackAsync(entityId);

        await harness.First.UpdateAsync(entityId, Host("prod-db", notes: "second"));
        await harness.SettleAsync();

        harness.Server.RowCount.ShouldBe(1);
        (await harness.First.FindAsync(entityId)).Secret.Notes.ShouldBe("second");
        (await harness.Second.FindAsync(entityId)).Secret.Notes.ShouldBe("second");
    }

    [Fact]
    public async Task AnUnacknowledgedCreateThatDidLand_IsDroppedQuietly()
    {
        // The same lost acknowledgement as above, but with no subsequent edit — which is the common case,
        // since a timeout is far more likely than a timeout followed by a change. The queued create meets
        // the server's own copy of itself, and the only correct answer is to stop trying to send it. In
        // particular this must not be reported as a conflict: there is nothing for a person to decide, and a
        // vault that produced a conflict notice every time a push timed out would train people to ignore
        // them.
        //
        // This is the one path that compares two decrypted items for equality, and the comparison has to go
        // through EqualityComparer rather than ==, because the reconciler is generic over the secret type
        // and == on a type parameter is reference equality.
        var entityId = await harness.First.CreateAsync(Host("prod-db", notes: "first"));

        await PushBehindTheEnginesBackAsync(entityId);

        await harness.SettleAsync();

        (await ConflictsAcrossDevicesAsync()).ShouldBeEmpty(
            "an item that came back exactly as it was sent is not something to arbitrate");

        // And the operation is gone rather than still being offered.
        (await harness.First.Outbox.TakeAsync(VaultId, 100, TestContext.Current.CancellationToken))
            .ShouldBeEmpty();

        harness.Server.RowCount.ShouldBe(1);
        (await harness.First.FindAsync(entityId)).HasUnsyncedChanges.ShouldBeFalse();
    }

    [Fact]
    public async Task AConcurrentWriteDuringAPush_IsNotSkipped()
    {
        // The cursor-gap hazard. The push response carries a cursor sitting after this client's own
        // changes; adopting it would skip anything another client committed at a lower sequence in the
        // window between this client's pull and its push. The engine keeps its own cursor instead.
        var mine = await harness.First.CreateAsync(Host("mine"));

        var theirs = Guid.CreateVersion7();
        harness.Server.OnPush = () => harness.Server.ExternalUpsert(theirs, ForeignPayload(), null);

        await harness.First.SyncAsync();

        var ids = (await harness.First.Items
            .ListAsync(VaultId, Contracts.SyncEntityType.Host, false, TestContext.Current.CancellationToken))
            .Select(item => item.EntityId)
            .ToArray();

        ids.ShouldContain(mine);
        ids.ShouldContain(theirs, "a change committed during the push was skipped");
    }

    [Fact]
    public async Task AForbiddenWrite_IsParkedRatherThanRetriedForever()
    {
        var entityId = await harness.First.CreateAsync(Host("prod-db"));

        harness.Server.DenyWrites = true;
        var report = await harness.First.SyncAsync();

        report.Parked.ShouldBe(1);
        (await harness.First.Outbox.ListParkedAsync(VaultId, TestContext.Current.CancellationToken))
            .ShouldHaveSingleItem().EntityId.ShouldBe(entityId);

        // A parked operation is not retried, so a second pass sends nothing.
        var pushes = harness.Server.PushCount;
        await harness.First.SyncAsync();
        harness.Server.PushCount.ShouldBe(pushes);

        (await harness.First.ConflictsAsync()).ShouldContain(c => c.Kind == ConflictKind.Rejected);
    }

    [Fact]
    public async Task AParkedChange_IsStillWhatTheUserSees()
    {
        // Hiding it because the server refused would show the old values and look like the edit was lost.
        var entityId = await harness.First.CreateAsync(Host("prod-db", notes: "mine"));

        harness.Server.DenyWrites = true;
        await harness.First.SyncAsync();

        var host = await harness.First.FindAsync(entityId);
        host.Secret.Notes.ShouldBe("mine");
        host.IsBlocked.ShouldBeTrue();
        host.HasUnsyncedChanges.ShouldBeTrue();
    }

    [Fact]
    public async Task ARekeyedVault_IsReportedRatherThanShowingAnEmptyList()
    {
        // After a rekey this client's grant is stale, so items pulled meanwhile cannot be read. Silently
        // showing nothing would look exactly like an empty vault.
        await harness.First.CreateAsync(Host("prod-db"));
        await harness.SettleAsync();

        harness.Server.KeyGeneration = 2;

        var report = await harness.First.SyncAsync();

        report.RekeyRequired.ShouldBeTrue();
        report.ServerKeyGeneration.ShouldBe(2u);
        report.NeedsAttention.ShouldBeTrue();
    }

    // ---- The overall property ----

    [Fact]
    public async Task AfterAnInterleavedSession_BothMachinesAgree()
    {
        // Convergence, over a fixed script that exercises creates, edits, a delete and a resurrection at
        // once. Two devices that ended up with different host lists would be the worst possible outcome
        // for a synced vault, and no single-case test rules it out.
        var shared = await harness.First.CreateAsync(Host("a-shared"));
        var doomed = await harness.First.CreateAsync(Host("b-doomed"));
        await harness.SettleAsync();

        await harness.First.UpdateAsync(shared, Host("a-shared", notes: "laptop note"));
        await harness.Second.UpdateAsync(shared, Host("a-shared", username: "desktop-user"));

        await harness.First.DeleteAsync(doomed);
        await harness.Second.UpdateAsync(doomed, Host("b-doomed", notes: "still wanted"));

        await harness.First.CreateAsync(Host("c-laptop-only"));
        await harness.Second.CreateAsync(Host("d-desktop-only"));

        await harness.SettleAsync();
        await harness.SettleAsync();

        var first = await harness.First.HostsSortedAsync();
        var second = await harness.Second.HostsSortedAsync();

        first.ShouldBe(second);
        first.Count.ShouldBe(4);

        first.Select(host => host.Label).ShouldBe(
            ["a-shared", "b-doomed (restored)", "c-laptop-only", "d-desktop-only"]);

        // The merged host kept both sides' contributions.
        var merged = first.Single(host => string.Equals(host.Label, "a-shared", StringComparison.Ordinal));
        merged.Notes.ShouldBe("laptop note");
        merged.Username.ShouldBe("desktop-user");

        // And nothing is still queued anywhere.
        (await harness.First.Outbox.TakeAsync(VaultId, 100, TestContext.Current.CancellationToken))
            .ShouldBeEmpty();
        (await harness.Second.Outbox.TakeAsync(VaultId, 100, TestContext.Current.CancellationToken))
            .ShouldBeEmpty();
    }

    // ---- Helpers ----

    private static Contracts.EncryptedPayload ForeignPayload() =>
        // Deliberately not decryptable: this stands in for another user's item, and the point of the test
        // is only that the change is not skipped. Pulling never decrypts, so nothing here needs to open.
        new([1, 2, 3], [4, 5], Guid.CreateVersion7(), 1, Crypto.CryptoSpec.CurrentAadVersion);

    /// <summary>
    /// Sends a device's queued operation straight to the server, leaving the outbox row in place.
    /// </summary>
    /// <remarks>
    /// Reproduces the one situation the engine cannot reach on its own: a push that the server applied
    /// and whose answer never came back. That window is where an idempotency key either saves the newer
    /// edit or destroys it.
    /// </remarks>
    private async Task PushBehindTheEnginesBackAsync(Guid entityId)
    {
        var pending = await harness.First.Outbox.FindAsync(
            VaultId, Contracts.SyncEntityType.Host, entityId, TestContext.Current.CancellationToken);

        pending.ShouldNotBeNull();

        await harness.Server.SyncPushAsync(
            VaultId,
            new Contracts.SyncPushRequest(
            [
                new Contracts.SyncPushOperation(
                    pending.OperationId,
                    pending.EntityType,
                    pending.EntityId,
                    pending.Operation,
                    pending.ExpectedVersion,
                    pending.Payload,
                    pending.Fields),
            ]),
            TestContext.Current.CancellationToken);
    }

    private async Task<IReadOnlyList<ConflictKind>> ConflictsAcrossDevicesAsync()
    {
        var first = await harness.First.ConflictsAsync();
        var second = await harness.Second.ConflictsAsync();

        return [.. first.Concat(second).Select(conflict => conflict.Kind)];
    }

    private async Task<IReadOnlyList<string>> DiscardedValuesAsync()
    {
        var first = await harness.First.ConflictsAsync();
        var second = await harness.Second.ConflictsAsync();

        return
        [
            .. first.Concat(second)
                .Select(conflict => System.Text.Encoding.UTF8.GetString(conflict.Detail)),
        ];
    }
}