diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c11bf58..315fb48 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1632,6 +1632,112 @@ jobs: if: always() && github.event_name != 'pull_request' run: docker logout "$REGISTRY" + # Rolls ssh.dodotech.cloud onto the image the job above just pushed, by calling the deploy hook of the + # Spelunk stack that runs it. Without this nothing told the server a new image existed, so a commit with + # a migration in it reached the registry and stopped there, and the schema never moved. + # + # The migration itself is applied by the new container as it starts — Database:AutoMigrate, see + # Persistence.MigrateDodoDatabaseAsync — under an advisory lock, before its port opens. So triggering + # the deploy is what applies it, and the wait below is what proves it: /healthz/ready is unhealthy for + # as long as any migration the running image knows about is pending, and an image whose migration + # failed does not start at all. + # + # main only, like the :main tag the stack follows. A failure here fails the run, unlike Spelunk's own + # trigger: a server left behind its schema is the outage this job exists to prevent, and a green build + # over one is the quiet version of it. + # + # SPELUNK_DEPLOY_TOKEN is a repository secret (Settings -> Actions -> Secrets), sent as a header so it + # stays out of every proxy and runner log a URL would be written to. + deploy: + name: deploy ssh.dodotech.cloud + needs: [image] + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: [linux] + timeout-minutes: 15 + # A newer push to main cancels this run like any other (see concurrency at the top). That is safe: + # once the hook has answered, the rollout and its migration happen on the server, and the newer + # run's own deploy job does the waiting instead. + env: + DEPLOY_HOOK: https://api.spelunk.dodotech.cloud/api/hooks/deploy + SERVER: https://ssh.dodotech.cloud + steps: + # See the desktop job's step of the same name. + - name: ensure curl + run: | + set -eu + SUDO="" + [ "$(id -u)" -eq 0 ] || SUDO="sudo" + + if ! command -v curl >/dev/null 2>&1; then + echo "Installing curl" + if command -v apk >/dev/null 2>&1; then + $SUDO apk add --no-cache curl + elif command -v apt-get >/dev/null 2>&1; then + $SUDO apt-get update -qq + $SUDO apt-get install -y --no-install-recommends curl + elif command -v dnf >/dev/null 2>&1; then + $SUDO dnf install -y curl + else + echo "No apk, apt-get or dnf here, so curl cannot be installed from inside the job." >&2 + exit 1 + fi + fi + curl --version | head -1 + + - name: trigger the deploy + env: + SPELUNK_DEPLOY_TOKEN: ${{ secrets.SPELUNK_DEPLOY_TOKEN }} + run: | + set -eu + + if [ -z "${SPELUNK_DEPLOY_TOKEN:-}" ]; then + echo "SPELUNK_DEPLOY_TOKEN is empty or unset, so the server was not told about this image." >&2 + echo "Add it under Settings -> Actions -> Secrets (not Variables)." >&2 + exit 1 + fi + + code="$(curl -sS -o "$RUNNER_TEMP/deploy-response" -w '%{http_code}' -X POST \ + -H "Authorization: Bearer $SPELUNK_DEPLOY_TOKEN" \ + "$DEPLOY_HOOK" || echo 000)" + + case "$code" in + 200|202|204) echo "Deploy hook accepted (HTTP $code)." ;; + 401|404) + echo "Deploy hook rejected the token (HTTP $code): revoked or regenerated, or the stack is gone." >&2 + exit 1 ;; + 409) + echo "Deploy hook refused the token (HTTP 409): it is limited to a service the stack no longer has." >&2 + exit 1 ;; + *) + echo "Deploy hook answered HTTP $code:" >&2 + cat "$RUNNER_TEMP/deploy-response" >&2 || true + exit 1 ;; + esac + + # Ten minutes, polled every ten seconds. A rollout restarts the container, so a few failed probes + # in the middle are expected; what fails the job is never becoming ready again — a migration that + # is still pending, one that crashed the start, or a stack holding the deploy for its schedule. + - name: wait for the server to be ready + run: | + set -eu + deadline=$(( $(date +%s) + 600 )) + last="" + while [ "$(date +%s)" -lt "$deadline" ]; do + sleep 10 + last="$(curl -sS -m 10 -w ' (HTTP %{http_code})' "$SERVER/healthz/ready" 2>&1 || true)" + case "$last" in + "Healthy (HTTP 200)") + echo "$SERVER is ready, with no migration pending." + curl -sS -m 10 "$SERVER/api/v1/meta" || true + echo + exit 0 ;; + esac + echo "not ready yet: $last" + done + echo "$SERVER did not become ready within ten minutes. Last answer: $last" >&2 + echo "A pending migration is named in the container's log (PendingMigrations)." >&2 + exit 1 + # No job here publishes the desktop *release* channel, and there is not going to be one. The desktop # nightly job above publishes a different application — its own pack id, its own Velopack channel, its own # install directory and profile — and the distance between those two sentences is the whole design. What diff --git a/src/DodoSSH.Api/Dockerfile b/src/DodoSSH.Api/Dockerfile index 8b4f826..3a06488 100644 --- a/src/DodoSSH.Api/Dockerfile +++ b/src/DodoSSH.Api/Dockerfile @@ -139,7 +139,8 @@ EXPOSE 8080 # without are DODOSSH_ConnectionStrings__Postgres, DODOSSH_Oidc__Authority and — on more # than one node — DODOSSH_Sync__CursorSigningKey. # -# Nothing migrates the database from in here. That is the API's own design: it fails -# /healthz/ready while a migration is pending and names it, so the schema is applied by -# `dotnet ef database update` alongside the deployment rather than by a racing container. +# The process migrates the database itself as it starts, before its port opens and under an +# advisory lock so two replicas cannot race (Database:AutoMigrate, on by default). Leave it on +# for ssh.dodotech.cloud: the CI deploy job relies on it. Turned off, /healthz/ready fails +# while a migration is pending and names it, and something else has to apply the schema. ENTRYPOINT ["dotnet", "DodoSSH.Api.dll"]