DodoSSH/tests/DodoSSH.Client.Ssh.Tests/SshServerFixture.cs

using System.Net.Sockets;
using System.Security.Cryptography;
using System.Text;
using DotNet.Testcontainers.Builders;
using DotNet.Testcontainers.Containers;
using Xunit;

namespace DodoSSH.Client.Ssh.Tests;

/// <summary>
/// A real OpenSSH server in a container, with password and public-key auth both enabled.
/// </summary>
/// <remarks>
/// <para>
/// One container per assembly. Everything the SSH layer needs to be right about — PTY
/// allocation, window-change requests, host key encoding, key formats — is behaviour of a real
/// sshd, and none of it can be established against a mock.
/// </para>
/// <para>
/// The image is Alpine-based, so <c>stty</c> comes from busybox. Its <c>stty size</c> prints
/// "rows cols", which is what the resize assertions read.
/// </para>
/// </remarks>
public sealed class SshServerFixture : IAsyncLifetime
{
    /// <summary>The account tests authenticate as.</summary>
    public const string Username = "dodo";

    /// <summary>Password for password authentication.</summary>
    public const string Password = "correct-horse-battery-staple";

    /// <summary>
    /// The port sshd listens on <em>inside</em> the container, rather than the one it is published on.
    /// </summary>
    /// <remarks>
    /// What anything reaching this server from within the container's own network namespace has to use —
    /// which includes a forward opened on a connection to it. <see cref="Port"/> is the mapped port on the
    /// host and is not listening in there. See <c>LoopbackProxyTests</c>, where the difference is the whole
    /// shape of the test.
    /// </remarks>
    public const int InternalPort = SshPort;

    private const int SshPort = 2222;

    /// <summary>
    /// How many connections in a row the server has to answer before this fixture calls it ready.
    /// </summary>
    /// <remarks>
    /// Twenty-five, and the number is measured rather than picked. Probing a fresh container 200 times with
    /// penalties left at the image's default, the first <c>Not allowed at this time</c> came back at probe
    /// 18 and 183 of the 200 were refused; with <c>PerSourcePenalties no</c> applied, none of 200 were. Ten
    /// was tried first and is useless — it sits below the threshold, so the guard passed happily against a
    /// server that was still penalising. See <see cref="WaitUntilServingAsync"/>.
    /// </remarks>
    private const int RequiredStreak = 25;

    /// <summary>How long to keep trying before giving up on the server entirely.</summary>
    private static readonly TimeSpan ReadyTimeout = TimeSpan.FromSeconds(60);

    private readonly SemaphoreSlim sftpGate = new(1, 1);

    private IContainer? container;
    private ISftpSession? sftp;

    /// <summary>Host port the container's sshd is published on.</summary>
    public ushort Port => container!.GetMappedPublicPort(SshPort);

    /// <summary>Host the container is reachable at.</summary>
    public string Host => container!.Hostname;

    /// <summary>An RSA key pair whose public half is authorized on the server.</summary>
    public RSA ClientKey { get; } = RSA.Create(3072);

    /// <inheritdoc />
    public async ValueTask InitializeAsync()
    {
        container = new ContainerBuilder("linuxserver/openssh-server:latest")
            .WithEnvironment("PUID", "1000")
            .WithEnvironment("PGID", "1000")
            .WithEnvironment("USER_NAME", Username)
            .WithEnvironment("USER_PASSWORD", Password)
            .WithEnvironment("PASSWORD_ACCESS", "true")
            .WithEnvironment("SUDO_ACCESS", "false")
            .WithEnvironment("PUBLIC_KEY", ExportOpenSshPublicKey(ClientKey))
            .WithPortBinding(SshPort, assignRandomHostPort: true)

            // The entrypoint generates host keys, rewrites sshd_config and installs the authorized
            // key before sshd is usable, so a published port is not readiness. Both conditions are
            // needed: the log line proves the authorized key was installed (a connection accepted
            // before that fails public-key auth), and the port check proves sshd is actually
            // accepting. The message is this image's own wording — "Server listening on" is
            // OpenSSH's and never appears here, which is a wait that hangs rather than fails.
            .WithWaitStrategy(Wait.ForUnixContainer()
                .UntilMessageIsLogged("Public key from env variable added")
                .UntilCommandIsCompleted("sh", "-c", $"netstat -ltn | grep -q ':{SshPort}'"))

            .Build();

        await container.StartAsync();
        await ReconfigureAsync();
        await WaitUntilServingAsync();
    }

    /// <summary>
    /// Turns off the hardening this suite trips over, and makes the running server re-read its config.
    /// </summary>
    /// <remarks>
    /// <para>
    /// ◆ <b><c>PerSourcePenalties no</c> is the fix for the flake this suite had for months, and the other
    /// two settings here are not.</b> OpenSSH 9.8 added per-source penalties and 10.x has them on by
    /// default; this image runs 10.3. A source address that keeps disconnecting without authenticating is
    /// penalised, and while the penalty holds every connection from it is answered with the clear-text line
    /// <c>Not allowed at this time</c> and then closed.
    /// </para>
    /// <para>
    /// <b>This suite generates exactly that traffic, by design.</b> This client's first contact with an
    /// unknown host is a connection deliberately refused at the host key — which is a disconnect with no
    /// authentication attempt — and several tests do nothing else:
    /// <c>RefusingTheHostKey_AbortsTheConnection</c>, <c>AnUntrustedHost_IsRefusedExactlyAsAShellWouldBe</c>,
    /// and every helper that learns a host key by being turned away first. Enough of them close together and
    /// sshd stops talking to the test host altogether, for a while, and then starts again.
    /// </para>
    /// <para>
    /// From the client that is <c>SshConnectionException: The connection was closed by the remote host</c>
    /// within milliseconds — no banner, nothing to say which of the many reasons it was. It hits whichever
    /// class is running when the penalty lands and spares the rest, which is why it read as random and why
    /// the class it hit lost <em>every</em> connection it made rather than a random few. The one test in that
    /// class that expects a refusal passed throughout, for the wrong reason.
    /// </para>
    /// <para>
    /// ◆ <b>Two earlier diagnoses were wrong, and are recorded here so they are not tried again.</b>
    /// <c>MaxStartups</c> was blamed on the reasoning that xUnit runs test classes in parallel, so ten
    /// unauthenticated connections would be in flight at once — but every class that touches this server
    /// shares <see cref="SshCollection"/>, and xUnit's unit of parallelism is the collection, so they run one
    /// after another and never have more than a connection or two open. The reload window was blamed next,
    /// and a wait for the banner to answer was written and removed as unproven; it was unproven because the
    /// banner does answer, right up until the penalty lands.
    /// </para>
    /// <para>
    /// The line is appended rather than replaced in place, unlike the two below it, because the image's
    /// config does not mention the keyword at all — there is no line to replace, and sshd takes the first
    /// value it finds for a keyword that appears more than once.
    /// </para>
    /// <para>
    /// ◆ <b><c>AllowTcpForwarding</c> is what a dynamic forward needs</b>, and the image ships it off as
    /// hardening. Without it a forward opens perfectly happily — a local listener asks the server nothing —
    /// and then every connection through it is refused when the channel is opened. SSH.NET reports that as
    /// <c>SOCKS5: General failure</c>, which names neither the server nor the setting, and is what the first
    /// run of <c>LoopbackProxyTests</c> collected. That suite is also the alarm if this method ever silently
    /// stops working.
    /// </para>
    /// <para>
    /// <c>MaxStartups</c> is raised for the reason it should have been in the first place rather than as a
    /// fix for anything: the compiled-in default refuses connections at random past ten unauthenticated ones
    /// in flight, and a throttle is hardening a test server has no business reproducing. It is kept, not
    /// because it was ever shown to matter here, but because removing it would be a second change riding
    /// along with this one.
    /// </para>
    /// <para>
    /// Both are replaced in place rather than appended, because sshd_config takes the <em>first</em> value
    /// it finds for a keyword: an appended line would be dead the day the image ships an uncommented one of
    /// its own.
    /// </para>
    /// <para>
    /// ◆ <b><c>/config/sshd/sshd_config</c>, and there are two.</b> The image also carries
    /// <c>/etc/ssh/sshd_config</c>, which looks like the file to patch, reads identically, and is not the one
    /// the running server was started with — patching it changes the text and nothing else, which is a fix
    /// that appears to work and leaves the failure exactly where it was.
    /// </para>
    /// <para>
    /// ◆ <b>Patched after boot and reloaded, rather than injected before it — which was tried and does not
    /// work.</b> This image family runs <c>/custom-cont-init.d</c> scripts, which look like the right hook
    /// and are not: the container's own log puts <c>sshd is listening on port 2222</c> <em>before</em>
    /// <c>[custom-init] Files found, executing</c>, so a script there edits a file the running server has
    /// already read. It leaves a config that greps correctly and a server behaving as though it had never
    /// been touched — the same trap as the wrong file, one layer up. Measured from the log, after a version
    /// of this fixture did exactly that and failed twenty-eight tests.
    /// </para>
    /// </remarks>
    private async Task ReconfigureAsync()
    {
        var result = await container!.ExecAsync([
            "sh",
            "-c",
            "sed -i 's/^AllowTcpForwarding no/AllowTcpForwarding yes/' /config/sshd/sshd_config"
                + " && sed -i 's/^#*MaxStartups .*/MaxStartups 200/' /config/sshd/sshd_config"
                + " && printf '\\nPerSourcePenalties no\\n' >> /config/sshd/sshd_config"
                + " && pkill -HUP sshd",
        ]);

        if (result.ExitCode != 0)
        {
            throw new InvalidOperationException(
                $"Could not reconfigure the test server: {result.Stderr}");
        }
    }

    /// <summary>
    /// Blocks until the server answers <see cref="RequiredStreak"/> connections in a row with its banner.
    /// </summary>
    /// <remarks>
    /// <para>
    /// ◆ <b>This is a guard rather than a wait, and what it guards against is
    /// <c>PerSourcePenalties</c> coming back.</b> Reconfiguring above turns it off; this proves it is off,
    /// immediately and by name, instead of letting the suite discover it later as an unrelated-looking
    /// failure in whichever class happened to be running.
    /// </para>
    /// <para>
    /// <b>Consecutive, and deliberately with no pause between them.</b> Each probe opens a connection, reads
    /// the identification string and disconnects without authenticating — which is exactly the shape of
    /// connection <c>PerSourcePenalties</c> punishes, and exactly what this suite does all day: a first
    /// contact with an unknown host is a connection this client deliberately refuses at the host key.
    /// <see cref="RequiredStreak"/> back to back is therefore not a soak test, it is the specific
    /// provocation, sized above the measured threshold on purpose, and it costs well under a second when the
    /// setting is off.
    /// </para>
    /// <para>
    /// It is also the one check that can tell a listening socket from a running server. The container's own
    /// readiness — a log line and <c>netstat</c> showing <c>:2222</c> — passes on a container whose sshd has
    /// gone: the socket is published by a host-side proxy that accepts before it has anything to forward to,
    /// so a dead server presents as a connection accepted and closed rather than as one refused.
    /// </para>
    /// <para>
    /// Probed from the host rather than with <c>docker exec</c>, deliberately: that is the path the tests
    /// take, proxy included, and penalties are counted per source address — from inside the container the
    /// source would be the loopback rather than the address every test connects from.
    /// </para>
    /// </remarks>
    private async Task WaitUntilServingAsync()
    {
        // TimeProvider.System rather than DateTimeOffset.UtcNow, which this repository bans so that time can
        // be faked — and rather than a fake, because what is being waited on is a real container starting.
        var deadline = TimeProvider.System.GetUtcNow() + ReadyTimeout;
        var streak = 0;
        var last = "no probe ran";

        while (streak < RequiredStreak)
        {
            if (TimeProvider.System.GetUtcNow() >= deadline)
            {
                throw new InvalidOperationException(
                    $"The test server did not answer {RequiredStreak} connections in a row within "
                    + $"{ReadyTimeout}. The last probe said: {last}. If it says \"Not allowed at this "
                    + "time\", sshd is penalising this source address and PerSourcePenalties is no longer "
                    + "being turned off — see ReconfigureAsync.");
            }

            var (answered, what) = await ProbeAsync();
            last = what;

            if (answered)
            {
                streak++;
                continue;
            }

            // Only pause when it is not working. Back-to-back probes are the point while they succeed;
            // hammering a server that has not finished starting is just noise.
            streak = 0;
            await Task.Delay(TimeSpan.FromMilliseconds(200));
        }
    }

    /// <summary>Opens a socket and reads far enough to see OpenSSH's identification string.</summary>
    /// <remarks>
    /// The description comes back with the answer because the interesting failures are not exceptions. A
    /// penalised source is told <c>Not allowed at this time</c> in clear text before the socket closes, and
    /// a suite that only knew "no banner" would have to go and find that out again — which is what happened
    /// the first time, at some length.
    /// </remarks>
    private async Task<(bool Answered, string What)> ProbeAsync()
    {
        try
        {
            using var probe = new TcpClient();
            using var timeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));

            await probe.ConnectAsync(Host, Port, timeout.Token);

            var buffer = new byte[64];
            var read = await probe.GetStream().ReadAtLeastAsync(
                buffer, 4, throwOnEndOfStream: false, timeout.Token);

            var answered = read >= 4 && "SSH-"u8.SequenceEqual(buffer.AsSpan(0, 4));

            return (
                answered,
                answered
                    ? "SSH-"
                    : $"{read} bytes: "
                        + Encoding.ASCII.GetString(buffer, 0, Math.Max(read, 0)).ReplaceLineEndings(" "));
        }
        catch (Exception exception) when (exception is SocketException or OperationCanceledException or IOException)
        {
            return (false, $"{exception.GetType().Name}: {exception.Message}");
        }
    }

    /// <summary>
    /// Adds one <c>authorized_keys</c> line to the account tests authenticate as.
    /// </summary>
    /// <remarks>
    /// <para>
    /// The fixture's own key arrives through the image's <c>PUBLIC_KEY</c> variable, which takes one. This
    /// is for the case that needs a second: proving a key <em>this client generated</em> authenticates
    /// against a real sshd, which is the only test that can establish the hand-written
    /// <c>openssh-key-v1</c> encoding is right. A parser accepting the file is weaker — SSH.NET could be
    /// forgiving about something OpenSSH is not.
    /// </para>
    /// <para>
    /// sshd re-reads <c>authorized_keys</c> on each authentication attempt, so nothing has to be restarted.
    /// </para>
    /// </remarks>
    public async ValueTask AuthorizeAsync(string publicKeyLine, CancellationToken cancellationToken)
    {
        ArgumentException.ThrowIfNullOrWhiteSpace(publicKeyLine);

        // Single-quoted in the shell and the line is base64 plus an algorithm name and a comment, so there
        // is nothing in it a quote could end. Asserted rather than assumed all the same: a silent failure
        // here would show up as an authentication error in a test whose subject is the key encoding, which
        // is the most misleading way for this to break.
        var result = await container!.ExecAsync(
            ["sh", "-c", $"echo '{publicKeyLine.Trim()}' >> /config/.ssh/authorized_keys"],
            cancellationToken);

        if (result.ExitCode != 0)
        {
            throw new InvalidOperationException(
                $"Could not install the public key in the container: {result.Stderr}");
        }
    }

    /// <summary>
    /// One file-transfer session, opened on first use and shared by every test that wants one.
    /// </summary>
    /// <remarks>
    /// <para>
    /// Shared rather than opened per test. This was once explained as a way of staying under the server's
    /// <c>MaxStartups</c> throttle, on the belief that the suite ran its classes in parallel and made two
    /// handshakes per test — this client's first contact with an unknown host is a connection deliberately
    /// <em>refused</em> at the host key, so every session costs two. The parallelism was not real: every
    /// class here shares one collection and xUnit runs collections, not classes, in parallel. See
    /// <see cref="WaitUntilServingAsync"/>, which is where that mistake was found and what the failure it
    /// was blamed for turned out to be.
    /// </para>
    /// <para>
    /// It stays shared regardless, on the plainer argument: one session is enough, and a handshake per test
    /// would be seconds of the suite's runtime spent proving nothing this file has not already proved.
    /// </para>
    /// <para>
    /// Safe to share because an SFTP session holds no per-test state: every test here works in a directory
    /// named after itself. See <c>ISftpSession</c>, which is one channel and is used by one caller at a
    /// time.
    /// </para>
    /// </remarks>
    public async ValueTask<ISftpSession> SftpAsync(CancellationToken cancellationToken)
    {
        await sftpGate.WaitAsync(cancellationToken);

        try
        {
            if (sftp is not null)
            {
                return sftp;
            }

            var knownHosts = new InMemoryKnownHostStore();
            var factory = new SshNetConnectionFactory(knownHosts);

            var request = new SshConnectionRequest(
                Host, Port, Username, new SshPasswordCredential(Password));

            try
            {
                // Learned by being refused, which is the only way this client learns a host key.
                return sftp = await factory.OpenSftpAsync(request, cancellationToken);
            }
            catch (SshHostKeyUnknownException unknown)
            {
                await knownHosts.TrustAsync(unknown.Presentation, cancellationToken);
            }

            return sftp = await factory.OpenSftpAsync(request, cancellationToken);
        }
        finally
        {
            sftpGate.Release();
        }
    }

    /// <inheritdoc />
    public async ValueTask DisposeAsync()
    {
        if (sftp is not null)
        {
            await sftp.DisposeAsync();
        }

        sftpGate.Dispose();

        if (container is not null)
        {
            await container.DisposeAsync();
        }

        ClientKey.Dispose();
    }

    /// <summary>
    /// Renders an RSA public key in the single-line <c>authorized_keys</c> format.
    /// </summary>
    /// <remarks>
    /// Hand-encoded because there is no BCL helper. The SSH wire format is a sequence of
    /// length-prefixed strings: the algorithm name, then the exponent, then the modulus — both
    /// as signed big-endian integers, which is why a leading zero byte is prepended when the
    /// high bit is set. Getting that wrong yields a key sshd silently ignores.
    /// </remarks>
    private static string ExportOpenSshPublicKey(RSA rsa)
    {
        var parameters = rsa.ExportParameters(includePrivateParameters: false);

        using var blob = new MemoryStream();
        WriteSshString(blob, "ssh-rsa"u8.ToArray());
        WriteSshMpint(blob, parameters.Exponent!);
        WriteSshMpint(blob, parameters.Modulus!);

        return $"ssh-rsa {Convert.ToBase64String(blob.ToArray())} dodossh-test";
    }

    private static void WriteSshString(Stream destination, byte[] value)
    {
        Span<byte> length = stackalloc byte[4];
        System.Buffers.Binary.BinaryPrimitives.WriteUInt32BigEndian(length, (uint)value.Length);
        destination.Write(length);
        destination.Write(value);
    }

    private static void WriteSshMpint(Stream destination, byte[] value)
    {
        // Signed big-endian: a high bit set would otherwise read as negative.
        if (value.Length > 0 && (value[0] & 0x80) != 0)
        {
            var padded = new byte[value.Length + 1];
            value.CopyTo(padded, 1);
            WriteSshString(destination, padded);
            return;
        }

        WriteSshString(destination, value);
    }
}

/// <summary>Shares one SSH server across every test class in the assembly.</summary>
[CollectionDefinition(Name)]
public sealed class SshCollection : ICollectionFixture<SshServerFixture>
{
    /// <summary>Collection name.</summary>
    public const string Name = "ssh";
}