Merge 82104d7f5f into 86c45d1d5f

* Update renovate syntax
* Update to Go 1.22.3

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

.github/actions/artifact_download/action.yml

@@ -16,11 +16,11 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install unzip
+    - name: Install 7zip
       uses: ./.github/actions/setup_bazel_nix
       with:
         nixTools: |
-          unzip
+          _7zz
 
     - name: Create temporary directory
       id: tempdir
@@ -37,4 +37,4 @@ runs:
       shell: bash
       run: |
         mkdir -p ${{ inputs.path }}
-        unzip -P '${{ inputs.encryptionSecret }}' -qq -d ${{ inputs.path }} ${{ steps.tempdir.outputs.directory }}/archive.zip
+        7zz x -p'${{ inputs.encryptionSecret }}' -t7z -o"${{ inputs.path }}" ${{ steps.tempdir.outputs.directory }}/archive.7z

.github/actions/artifact_upload/action.yml

@@ -22,11 +22,11 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install zip
+    - name: Install 7zip
       uses: ./.github/actions/setup_bazel_nix
       with:
         nixTools: |
-          zip
+          _7zz
 
     - name: Create temporary directory
       id: tempdir
@@ -37,10 +37,8 @@ runs:
       shell: bash
       run: |
         shopt -s extglob
-
         paths="${{ inputs.path }}"
         paths=${paths%$'\n'} # Remove trailing newline
-
         # Check if any file matches the given pattern(s).
         something_exists=false
         for pattern in ${paths}
@@ -49,7 +47,6 @@ runs:
             something_exists=true
           fi
         done
-
         # Create an archive if files exist.
         # Don't create an archive file if no files are found
         # and warn.
@@ -58,11 +55,10 @@ runs:
           echo "::warning:: No files/directories found with the provided path(s): ${paths}. No artifact will be uploaded."
           exit 0
         fi
-
         for target in ${paths}
         do
           pushd "$(dirname "${target}")" || exit 1
-          zip -e -P '${{ inputs.encryptionSecret }}' -r "${{ steps.tempdir.outputs.directory }}/archive.zip" "$(basename "${target}")"
+          7zz a -p'${{ inputs.encryptionSecret }}' -t7z -ms=on -mhe=on "${{ steps.tempdir.outputs.directory }}/archive.7z" "$(basename "${target}")"
           popd || exit 1
         done
 
@@ -70,7 +66,7 @@ runs:
       uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       with:
         name: ${{ inputs.name }}
-        path: ${{ steps.tempdir.outputs.directory }}/archive.zip
+        path: ${{ steps.tempdir.outputs.directory }}/archive.7z
         retention-days: ${{ inputs.retention-days }}
         if-no-files-found: ignore
         overwrite: ${{ inputs.overwrite }}

.github/actions/constellation_create/action.yml

@@ -262,7 +262,7 @@ runs:
         mkdir to-zip
         cp -r constellation-terraform to-zip
         cp -r constellation-iam-terraform to-zip
-        rm to-zip/constellation-terraform/plan.zip
+        rm -f to-zip/constellation-terraform/plan.zip
         rm -rf to-zip/constellation-terraform/.terraform to-zip/constellation-iam-terraform/.terraform
 
     - name: Upload terraform state

.github/actions/e2e_cleanup_timeframe/action.yml

@@ -31,11 +31,11 @@ runs:
       with:
         service_account: "destroy-e2e@constellation-e2e.iam.gserviceaccount.com"
 
-    - name: Install unzip
+    - name: Install 7zip
       uses: ./.github/actions/setup_bazel_nix
       with:
         nixTools: |
-          unzip
+          _7zz
     - name: Run cleanup
       run: ./.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh
       shell: bash

.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh

@@ -3,7 +3,7 @@
 # get_e2e_test_ids_on_date gets all workflow IDs of workflows that contain "e2e" on a specific date.
 function get_e2e_test_ids_on_date {
   ids="$(gh run list --created "$1" --status failure --json createdAt,workflowName,databaseId --jq '.[] | select(.workflowName | contains("e2e") and (contains("MiniConstellation") | not)) | .databaseId' -L1000 -R edgelesssys/constellation || exit 1)"
-  echo "$ids"
+  echo "${ids}"
 }
 
 # download_tfstate_artifact downloads all artifacts matching the pattern terraform-state-* from a given workflow ID.
@@ -13,7 +13,7 @@ function download_tfstate_artifact {
 
 # delete_resources runs terraform destroy on the constellation-terraform subfolder of a given folder.
 function delete_resources {
-  if [ -d "$1/constellation-terraform" ]; then
+  if [[ -d "$1/constellation-terraform" ]]; then
     cd "$1/constellation-terraform" || exit 1
     terraform init > /dev/null || exit 1 # first, install plugins
     terraform destroy -auto-approve || exit 1
@@ -23,7 +23,7 @@ function delete_resources {
 
 # delete_iam_config runs terraform destroy on the constellation-iam-terraform subfolder of a given folder.
 function delete_iam_config {
-  if [ -d "$1/constellation-iam-terraform" ]; then
+  if [[ -d "$1/constellation-iam-terraform" ]]; then
     cd "$1/constellation-iam-terraform" || exit 1
     terraform init > /dev/null || exit 1 # first, install plugins
     terraform destroy -auto-approve || exit 1
@@ -32,12 +32,12 @@ function delete_iam_config {
 }
 
 # check if the password for artifact decryption was given
-if [[ -z $ENCRYPTION_SECRET ]]; then
+if [[ -z ${ENCRYPTION_SECRET} ]]; then
   echo "ENCRYPTION_SECRET is not set. Please set an environment variable with that secret."
   exit 1
 fi
 
-artifact_pwd=$ENCRYPTION_SECRET
+artifact_pwd=${ENCRYPTION_SECRET}
 
 shopt -s nullglob
 
@@ -46,9 +46,9 @@ end_date=$(date --date "-7 day" "+%Y-%m-%d")
 dates_to_clean=()
 
 # get all dates of the last week
-while [[ $end_date != "$start_date" ]]; do
-  dates_to_clean+=("$end_date")
-  end_date=$(date --date "$end_date +1 day" "+%Y-%m-%d")
+while [[ ${end_date} != "${start_date}" ]]; do
+  dates_to_clean+=("${end_date}")
+  end_date=$(date --date "${end_date} +1 day" "+%Y-%m-%d")
 done
 
 echo "[*] retrieving run IDs for cleanup"
@@ -65,33 +65,33 @@ mapfile -td " " database_ids < <(echo "${database_ids[@]}")
 
 echo "[*] downloading terraform state artifacts"
 for id in "${database_ids[@]}"; do
-  if [[ $id == *[^[:space:]]* ]]; then
-    echo "    downloading from workflow $id"
-    download_tfstate_artifact "$id"
+  if [[ ${id} == *[^[:space:]]* ]]; then
+    echo "    downloading from workflow ${id}"
+    download_tfstate_artifact "${id}"
   fi
 done
 
 echo "[*] extracting artifacts"
 for directory in ./terraform-state-*; do
-  echo "    extracting $directory"
+  echo "    extracting ${directory}"
 
   # extract and decrypt the artifact
-  unzip -d "${directory}" -P "$artifact_pwd" "$directory/archive.zip" > /dev/null || exit 1
+  7zz x -t7z -p"${artifact_pwd}" -o"${directory}" "${directory}/archive.7z" > /dev/null || exit 1
 done
 
 # create terraform caching directory
-mkdir "$HOME/tf_plugin_cache"
-export TF_PLUGIN_CACHE_DIR="$HOME/tf_plugin_cache"
-echo "[*] created terraform cache directory $TF_PLUGIN_CACHE_DIR"
+mkdir "${HOME}/tf_plugin_cache"
+export TF_PLUGIN_CACHE_DIR="${HOME}/tf_plugin_cache"
+echo "[*] created terraform cache directory ${TF_PLUGIN_CACHE_DIR}"
 
 echo "[*] deleting resources"
 for directory in ./terraform-state-*; do
-  echo "    deleting resources in $directory"
-  delete_resources "$directory"
-  echo "    deleting IAM configuration in $directory"
-  delete_iam_config "$directory"
-  echo "    deleting directory $directory"
-  rm -rf "$directory"
+  echo "    deleting resources in ${directory}"
+  delete_resources "${directory}"
+  echo "    deleting IAM configuration in ${directory}"
+  delete_iam_config "${directory}"
+  echo "    deleting directory ${directory}"
+  rm -rf "${directory}"
 done
 
 exit 0

.github/actions/update_tfstate/action.yml

@@ -1,5 +1,5 @@
 name: Update TFState
-description: "Update the terraform state artifact."
+description: "Update the terraform state artifact. We use this to either delete an artifact if the e2e test was cleaned up successfully or to update the artifact with the latest terraform state."
 
 inputs:
   name:
@@ -11,16 +11,12 @@ inputs:
   encryptionSecret:
     description: "The encryption secret for the artifacts."
     required: true
-  skipDeletion:
-    description: "Don't try to delete the artifact before updating. You should only use this if you know that no artifact exists."
-    default: "false"
-    required: false
 
 runs:
   using: "composite"
   steps:
-    - name: Check if tfstate should be deleted
-      if: always() && inputs.skipDeletion == 'false'
+    - name: Check if uploaded tfstate can be deleted
+      if: always()
       shell: bash
       run: |
         if [[ ! -d constellation-terraform ]] && [[ ! -d constellation-iam-terraform ]]; then
@@ -30,14 +26,14 @@ runs:
         fi
 
     - name: Delete tfstate artifact if necessary
-      if: always() && env.DELETE_TF_STATE == 'true' && inputs.skipDeletion == 'false'
+      if: always() && env.DELETE_TF_STATE == 'true'
       uses: ./.github/actions/artifact_delete
       with:
         name: ${{ inputs.name }}
         workflowID: ${{ inputs.runID }}
 
-    - name: Prepare terraform state folders
-      if: always()
+    - name: Prepare left over terraform state folders
+      if: always() && env.DELETE_TF_STATE == 'false'
       shell: bash
       run: |
         rm -rf to-zip/*

.github/actions/upload_terraform_module/action.yml

@@ -23,4 +23,4 @@ runs:
     - name: Cleanup Terraform module dir
       shell: bash
       run: |
-        rm -r terraform-module terraform-module.zip
+        rm -f terraform-module terraform-module.zip

.github/actions/versionsapi/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.2@sha256:c4fb952e712efd8f787bcd8e53fd66d1d83b7dc26adabc218e9eac1dbf776bdf as builder
+FROM golang:1.22.3@sha256:b1e05e2c918f52c59d39ce7d5844f73b2f4511f7734add8bb98c9ecdd4443365 as builder
 
 # Download project root dependencies
 WORKDIR /workspace

.github/workflows/build-ccm-gcp.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Setup Go environment
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: "1.22.2"
+          go-version: "1.22.3"
           cache: false
 
       - name: Install Crane

.github/workflows/build-os-image-scheduled.yml

@@ -69,7 +69,7 @@ jobs:
       - name: Setup Go environment
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: "1.22.2"
+          go-version: "1.22.3"
           cache: false
 
       - name: Determine version

.github/workflows/codeql.yml

@@ -40,7 +40,7 @@ jobs:
         if: matrix.language == 'go'
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: "1.22.2"
+          go-version: "1.22.3"
           cache: false
 
       - name: Initialize CodeQL

.github/workflows/e2e-upgrade.yml

@@ -513,7 +513,7 @@ jobs:
         run: |
           mkdir -p to-zip
           cp -r constellation-terraform to-zip
-          rm to-zip/constellation-terraform/plan.zip
+          rm -f to-zip/constellation-terraform/plan.zip
           rm -rf to-zip/constellation-terraform/.terraform
           cp -r constellation-iam-terraform to-zip
           rm -rf to-zip/constellation-iam-terraform/.terraform
@@ -542,7 +542,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
         uses: ./.github/actions/update_tfstate
         with:
-          name: terraform-state-${{ needs.create-cluster.outputs.e2e-name-prefix }} 
+          name: terraform-state-${{ needs.create-cluster.outputs.e2e-name-prefix }}
           runID: ${{ github.run_id }}
           encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 

.github/workflows/release.yml

@@ -233,7 +233,7 @@ jobs:
       - name: Setup Go environment
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: "1.22.2"
+          go-version: "1.22.3"
           cache: true
 
       - name: Build generateMeasurements tool

.github/workflows/test-operator-codegen.yml

@@ -28,7 +28,7 @@ jobs:
       - name: Setup Go environment
         uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
         with:
-          go-version: "1.22.2"
+          go-version: "1.22.3"
           cache: true
 
       - name: Run code generation

3rdparty/gcp-guest-agent/Dockerfile

@@ -6,7 +6,7 @@ RUN apt-get update && apt-get install -y \
     git
 
 # Install Go
-ARG GO_VER=1.22.2
+ARG GO_VER=1.22.3
 RUN wget -q https://go.dev/dl/go${GO_VER}.linux-amd64.tar.gz && \
     tar -C /usr/local -xzf go${GO_VER}.linux-amd64.tar.gz && \
     rm go${GO_VER}.linux-amd64.tar.gz

WORKSPACE.bazel

@@ -170,7 +170,7 @@ load("@io_bazel_rules_go//go:deps.bzl", "go_download_sdk", "go_register_toolchai
 go_download_sdk(
     name = "go_sdk",
     patches = ["//3rdparty/bazel/org_golang:go_tls_max_handshake_size.patch"],
-    version = "1.22.2",
+    version = "1.22.3",
 )
 
 go_rules_dependencies()

dev-docs/workflows/bump-go-version.md

@@ -1,4 +1,5 @@
 # Bump Go version
+
 `govulncheck` from the bazel `check` target will fail if our code is vulnerable, which is often the case when a patch version was released with security fixes.
 
 ## Steps
@@ -6,5 +7,13 @@
 Replace "1.xx.x" with the new version in [WORKSPACE.bazel](/WORKSPACE.bazel):
 
 ```starlark
-go_register_toolchains(version = "1.xx.x")
+load("@io_bazel_rules_go//go:deps.bzl", "go_download_sdk", "go_register_toolchains", "go_rules_dependencies")
+
+go_download_sdk(
+    name = "go_sdk",
+    patches = ["//3rdparty/bazel/org_golang:go_tls_max_handshake_size.patch"],
+    version = "1.xx.x", <--- Replace this one
+              ~~~~~~~~
+)
+
 ```

go.work

@@ -1,6 +1,6 @@
-go 1.22.2
+go 1.22.3
 
-toolchain go1.22.2
+toolchain go1.22.3
 
 use (
 	.

renovate.json5

@@ -42,44 +42,44 @@
       "prPriority": -30,
     },
     {
-      "matchPackagePatterns": ["^k8s.io", "^sigs.k8s.io"],
+      "matchDepPatterns": ["^k8s.io", "^sigs.k8s.io"],
       "groupName": "K8s dependencies",
     },
     {
-      "matchPackagePatterns": ["^go.etcd.io/etcd"],
+      "matchDepPatterns": ["^go.etcd.io/etcd"],
       "groupName": "etcd dependencies",
     },
     {
-      "matchPackagePatterns": ["^github.com/hashicorp/go-kms-wrapping"],
+      "matchDepPatterns": ["^github.com/hashicorp/go-kms-wrapping"],
       "groupName": "github.com/hashicorp/go-kms-wrapping",
     },
     {
-      "matchPackagePatterns": ["^github.com/aws/aws-sdk-go-v2"],
+      "matchDepPatterns": ["^github.com/aws/aws-sdk-go-v2"],
       "groupName": "AWS SDK",
       "prPriority": -10,
     },
     {
-      "matchPackagePatterns": [
+      "matchDepPatterns": [
         "^github.com/Azure/",
         "^github.com/AzureAD/microsoft-authentication-library-for-go",
       ],
       "groupName": "Azure SDK",
     },
     {
-      "matchPackagePatterns": ["^cloud.google.com/go"],
+      "matchDepPatterns": ["^cloud.google.com/go"],
       "groupName": "Google SDK",
     },
     {
-      "matchPackagePatterns": ["^google.golang.org/genproto"],
+      "matchDepPatterns": ["^google.golang.org/genproto"],
       "prPriority": -10,
     },
     {
-      "matchPackagePatterns": ["^libvirt.org/go"],
+      "matchDepPatterns": ["^libvirt.org/go"],
       "groupName": "libvirt.org/go",
     },
     {
       "matchManagers": ["bazelisk", "bazel", "bazel-module"],
-      "matchPackageNames": ["bazel", "io_bazel_rules_go", "bazel_gazelle"],
+      "matchDepNames": ["bazel", "io_bazel_rules_go", "bazel_gazelle"],
       "groupName": "bazel (core)",
     },
     {
@@ -105,14 +105,14 @@
       ],
     },
     {
-      "matchPackageNames": ["kubernetes/kubernetes"],
+      "matchDepNames": ["kubernetes/kubernetes"],
       // example match: v1.2.3 (1.2 -> compatibility, 3 -> patch)
       "versioning": "regex:^(?<compatibility>v?\\d+\\.\\d+\\.)(?<patch>\\d+)$",
       "groupName": "Kubernetes versions",
       "prPriority": 15,
     },
     {
-      "matchPackageNames": [
+      "matchDepNames": [
         "registry.k8s.io/provider-aws/cloud-controller-manager",
       ],
       // example match: v1.2.3 (1.2 -> compatibility, 3 -> patch)
@@ -121,7 +121,7 @@
       "prPriority": 15,
     },
     {
-      "matchPackageNames": [
+      "matchDepNames": [
         "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager",
         "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager",
       ],
@@ -131,7 +131,7 @@
       "prPriority": 15,
     },
     {
-      "matchPackageNames": [
+      "matchDepNames": [
         "docker.io/k8scloudprovider/openstack-cloud-controller-manager",
       ],
       // example match: v1.2.3 (1.2 -> compatibility, 3 -> patch)
@@ -140,14 +140,14 @@
       "prPriority": 15,
     },
     {
-      "matchPackageNames": ["registry.k8s.io/autoscaling/cluster-autoscaler"],
+      "matchDepNames": ["registry.k8s.io/autoscaling/cluster-autoscaler"],
       // example match: v1.2.3 (1.2 -> compatibility, 3 -> patch)
       "versioning": "regex:^(?<compatibility>v?\\d+\\.\\d+\\.)(?<patch>\\d+)$",
       "groupName": "K8s constrained GCP versions",
       "prPriority": 15,
     },
     {
-      "matchPackageNames": ["ghcr.io/edgelesssys/cloud-provider-gcp"],
+      "matchDepNames": ["ghcr.io/edgelesssys/cloud-provider-gcp"],
       // example match: v1.2.3 (1. -> compatibility, 2 -> minor, 3 -> patch)
       "versioning": "regex:^(?<compatibility>v\\d+\\.)(?<minor>\\d+)\\.(?<patch>\\d+)$",
       "groupName": "cloud-provider-gcp (K8s version constrained)",
@@ -166,7 +166,7 @@
       "prPriority": 20,
     },
     {
-      "matchPackageNames": [
+      "matchDepNames": [
         "registry.k8s.io/kas-network-proxy/proxy-agent",
         "registry.k8s.io/kas-network-proxy/proxy-server",
       ],
@@ -175,7 +175,7 @@
       "prPriority": 15,
     },
     {
-      "matchPackageNames": ["^k8s.io/client-go"],
+      "matchDepNames": ["^k8s.io/client-go"],
       "matchUpdateTypes": ["major"],
       "enabled": false,
     },
@@ -185,11 +185,11 @@
     },
     {
       "matchManagers": ["github-actions"],
-      "matchPackageNames": ["slsa-framework/slsa-github-generator"],
+      "matchDepNames": ["slsa-framework/slsa-github-generator"],
       "pinDigests": false,
     },
     {
-      "matchPackagePatterns": ["_(darwin|linux)_(arm64|amd64)$"],
+      "matchDepPatterns": ["_(darwin|linux)_(arm64|amd64)$"],
       "additionalBranchPrefix": "{{packageName}}-",
       "groupName": "{{packageName}}",
     },

rfc/016-node-access.md

@@ -0,0 +1,176 @@
+# Node Access
+
+## Background
+
+A production Constellation cluster is currently configured not to allow any kind of remote administrative access.
+This choice is deliberate: any mechanism for remote accesss can potentially be exploited, or may leak sensitive data.
+
+However, some operations on a Kubernetes cluster require some form of access to the nodes.
+A good class of examples are etcd cluster maintenance tasks, like backup and recovery, or emergency operations like removing a permanently failed member.
+Some kubeadm operations, like certificate rotation, also require some form of cluster access.
+
+While some tasks can be accomplished by DaemonSets, CronJobs and the like, relying on Kubernetes objects is insufficient.
+Executing commands in a Kubernetes pod may fail because Kubernetes is not healthy, etcd is bricked or the network is down.
+Administrative access to the nodes through a side channel would greatly help remediate, or at least debug, those situations.
+
+## Requirements
+
+Constellation admins can log into Constellation nodes for maintenance, subject to the following restrictions:
+
+* Access must be encrypted end-to-end to protect from CSP snooping.
+* Access must be possible even if the Kubernetes API server is down.
+
+Nice-to-have:
+
+* The method of access should not require long-term storage of a second secret.
+* The method of access should be time-limited.
+
+## Proposed Design
+
+Core to the proposal is [certificate-based authentication for OpenSSH](https://en.wikibooks.org/wiki/OpenSSH/Cookbook/Certificate-based_Authentication).
+We can derive a valid SSH key from the Constellation master secret.
+An OpenSSH server on the node accepts certificates issued by this CA key.
+Admins can derive the CA key from the master secret on demand, and issue certificates for arbitrary public keys.
+An example program is in the [Appendix](#appendix).
+
+### Key Details
+
+We use an HKDF to derive an ed25519 private key from the master secret.
+This private key acts as an SSH certificate authority, whose signed certs allow access to cluster nodes.
+Since the master secret is available to both the cluster owner and the nodes, no communication with the cluster is needed to mint valid certificates.
+The choice of curve allows to directly use the derived secret bytes as key.
+This makes the implementation deterministic, and thus the CA key recoverable.
+
+### Server-side Details
+
+An OpenSSH server is added to the node image software stack.
+It's configured with a `TrustedUserCAKeys` file that is empty on startup.
+After initialization, the bootstrapper fills that file with the derived CA's public key.
+All other means of authentication are disabled.
+
+### Client-side Details
+
+A new subcommand `constellation credentials ssh` is added to the CLI.
+It takes the master secret file and an SSH pub key file as arguments, and writes a certificate to stdout.
+Optional arguments may include principals or vailidity period.
+The implementation could roughly follow the PoC in the [Appendix](#appendix).
+
+## Security Considerations
+
+Exposing an additional service to the outside world increases the attack surface.
+We propose the following mitigations:
+
+1. The SSH port is only exposed to the VPC.
+   This restricts the attackers to malicious co-tenants and the CSP.
+   In an emergency, admins need to add a load balancer to be able to reach the nodes.
+2. A hardened OpenSSH config only allows the options strictly necessary for the scheme proposed here.
+   Authorized keys and passwords must be disabled.
+   Cipher suites should be restricted. etc.
+
+## Alternatives Considered
+
+### Enable Serial Console
+
+Serial consoles for cloud VMs are tunnelled through the CSP in the clear.
+To make this solution secure, an encrypted channel would need to be established on top of the serial connection.
+The author is not aware of any software providing such a channel.
+
+### SSH with Authorized Keys
+
+We could ask users to add a public key to their `constellation-conf.yaml` and add that to `/root/.ssh/authorized_keys` after joining.
+This would require the cluster owner to permanently manage a second secret, and there would be no built-in way to revoke access.
+
+### Debug Pod
+
+Some node administration tasks can be performed with a [debug pod].
+If privileged access is required, it's usually necessary to schedule a custom pod.
+This only works if the Kubernetes API server is still processing requests, pods can be scheduled on the target node and the network allows connecting to it.
+
+[debug pod]: https://kubernetes.io/docs/tasks/debug/debug-cluster/kubectl-node-debug/
+
+### Host an Admin API Server
+
+There are alternatives to SSH that allow fine-grained authorization of node operations.
+An example would be [SansShell], which verifies node access requests with a policy.
+Setting up such a tool requires a detailed understanding of the use cases, of which some might be hard to foresee.
+This may be better suited as an extension of the low-level emergency access mechanisms.
+
+[SansShell]: https://www.snowflake.com/blog/sansshell-local-host-agent/
+
+## Appendix
+
+A proof-of-concept implementation of the certificate generation.
+Constellation nodes would stop after deriving the CA public key.
+
+```golang
+package main
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"golang.org/x/crypto/hkdf"
+	"golang.org/x/crypto/ssh"
+)
+
+type secret struct {
+	Key  []byte `json:"key,omitempty"`
+	Salt []byte `json:"salt,omitempty"`
+}
+
+var permissions = ssh.Permissions{
+	Extensions: map[string]string{
+		"permit-port-forwarding": "yes",
+		"permit-pty":             "yes",
+	},
+}
+
+func main() {
+	masterSecret := flag.String("secret", "", "")
+	flag.Parse()
+
+	secretJSON, err := os.ReadFile(*masterSecret)
+	must(err)
+	var secret secret
+	must(json.Unmarshal(secretJSON, &secret))
+
+	hkdf := hkdf.New(sha256.New, secret.Key, secret.Salt, []byte("ssh-ca"))
+
+	_, priv, err := ed25519.GenerateKey(hkdf)
+	must(err)
+
+	ca, err := ssh.NewSignerFromSigner(priv)
+	must(err)
+
+	log.Printf("CA KEY: %s", string(ssh.MarshalAuthorizedKey(ca.PublicKey())))
+
+	buf, err := os.ReadFile(flag.Arg(0))
+	must(err)
+	pub, _, _, _, err := ssh.ParseAuthorizedKey(buf)
+	must(err)
+	certificate := ssh.Certificate{
+		Key:             pub,
+		CertType:        ssh.UserCert,
+		ValidAfter:      uint64(time.Now().Unix()),
+		ValidBefore:     uint64(time.Now().Add(24 * time.Hour).Unix()),
+		ValidPrincipals: []string{"root"},
+		Permissions:     permissions,
+	}
+	must(certificate.SignCert(rand.Reader, ca))
+
+	fmt.Printf("%s\n", string(ssh.MarshalAuthorizedKey(&certificate)))
+}
+
+func must(err error) {
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+```

terraform/infrastructure/aws/main.tf

@@ -213,5 +213,5 @@ module "jump_host" {
   ports                = [for port in local.load_balancer_ports : port.port]
   security_groups      = [aws_security_group.security_group.id]
   iam_instance_profile = var.iam_instance_profile_name_worker_nodes
-  additional_tags      = local.tags
+  additional_tags      = var.additional_tags
 }

terraform/infrastructure/aws/modules/jump_host/main.tf

@@ -27,7 +27,7 @@ resource "aws_instance" "jump_host" {
   vpc_security_group_ids = var.security_groups
 
   tags = merge(var.additional_tags, {
-    "Name" = "${var.base_name}-jump-host"
+    "Name" = "${var.base_name}-jump-host",
   })
 
   user_data = <<EOF

terraform/infrastructure/azure/main.tf

@@ -276,7 +276,7 @@ module "jump_host" {
   subnet_id      = azurerm_subnet.loadbalancer_subnet[0].id
   ports          = [for port in local.ports : port.port]
   lb_internal_ip = azurerm_lb.loadbalancer.frontend_ip_configuration[0].private_ip_address
-  tags           = local.tags
+  tags           = var.additional_tags
 }
 
 data "azurerm_subscription" "current" {

terraform/infrastructure/gcp/main.tf

@@ -240,7 +240,7 @@ module "jump_host" {
   base_name      = local.name
   zone           = var.zone
   subnetwork     = google_compute_subnetwork.vpc_subnetwork.id
-  labels         = local.labels
+  labels         = var.additional_labels
   lb_internal_ip = google_compute_address.loadbalancer_ip_internal[0].address
   ports          = [for port in local.control_plane_named_ports : port.port]
 }