ci: add e2e-test-release workflow

This workflow is used to run e2e tests in preparation to a release. It is triggered by the successful completion of the release workflow. Also trigger e2e-mini through the release workflow completion. This makes restarting the tests easier if they fail during release preparation. Co-authored-by: stdoutput <moritz.sanft@outlook.de>
2024-10-01 01:36:09 -04:00 · 2023-02-13 13:58:34 +01:00 · 2023-02-13 13:58:34 +01:00 · 4df33b93fe
commit 4df33b93fe
parent 2d41a19fbf
8 changed files with 378 additions and 109 deletions
--- a/.github/actions/constellation_create/action.yml
+++ b/.github/actions/constellation_create/action.yml
@ -75,7 +75,7 @@ runs:
      shell: bash
      if: inputs.existingConfig != 'true'
      run: |
-        if [[ ${{ inputs.kubernetesVersion != '' }} == true ]]; then
+        if [[ -n "${{ inputs.kubernetesVersion }}" ]]; then
          constellation config generate ${{ inputs.cloudProvider }} --kubernetes="${{ inputs.kubernetesVersion }}" --debug
        else
          constellation config generate ${{ inputs.cloudProvider }} --debug
@ -107,6 +107,10 @@ runs:
            (.provider | select(. | has(\"aws\")).aws.iamProfileWorkerNodes) = \"e2e_test_worker_node_instance_profile\"" \
          constellation-conf.yaml

+        if [[ -n "${{ inputs.kubernetesVersion }}" ]]; then
+          yq eval -i "(.kubernetesVersion) = \"${{ inputs.kubernetesVersion }}\"" constellation-conf.yaml
+        fi
+
    - name: Remove embedded measurements
      if: inputs.keepMeasurements == 'false'
      shell: bash
@ -128,6 +132,7 @@ runs:
          constellation-conf.yaml

    - name: Set image
+      if: inputs.osImage != ''
      shell: bash
      env:
        image: ${{ inputs.osImage }}
--- a/.github/actions/select_image/action.yml
+++ b/.github/actions/select_image/action.yml
@ -0,0 +1,79 @@
+name: Select image
+description: Resolve string presets and shortpaths to shortpaths only
+
+inputs:
+  osImage:
+    description: "Shortpath or main-debug or release-stable"
+    required: true
+
+outputs:
+  osImage:
+    description: "Shortpath of for input string, original input if that was already a shortpath"
+    value: ${{ steps.set-output.outputs.osImage }}
+  isDebugImage:
+    description: "Input represents a debug image or not"
+    value: ${{ steps.set-output.outputs.isDebugImage }}
+
+runs:
+  using: "composite"
+  steps:
+    - name: Login to AWS
+      uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
+        aws-region: eu-central-1
+
+    - name: Input is preset
+      id: input-is-preset
+      shell: bash
+      run: |
+        if [[ "${{ inputs.osImage }}" == "ref/main/stream/debug/?" || "${{ inputs.osImage }}" == "ref/release/stream/stable/?" ]]; then
+          echo "result=true" >> "$GITHUB_OUTPUT"
+        else
+          echo "result=false" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Separate ref and stream from matrix
+      if: steps.input-is-preset.outputs.result == 'true'
+      id: separate-ref-stream
+      env:
+        REFSTREAM: ${{ inputs.osImage }}
+      shell: bash
+      run: |
+        echo "ref=$(echo $REFSTREAM | cut -d/ -f2)" | tee -a "$GITHUB_OUTPUT"
+        echo "stream=$(echo $REFSTREAM | cut -d/ -f4)" | tee -a "$GITHUB_OUTPUT"
+
+
+    - name: Find latest image
+      if: steps.input-is-preset.outputs.result == 'true'
+      id: find-latest-image
+      uses: ./.github/actions/versionsapi
+      with:
+        command: latest
+        ref: ${{ steps.separate-ref-stream.outputs.ref == 'release' && '-' || steps.separate-ref-stream.outputs.ref }}
+        stream: ${{ steps.separate-ref-stream.outputs.stream }}
+
+    - name: Set outputs
+      id: set-output
+      shell: bash
+      run: |
+        if [[ ${{ steps.input-is-preset.outputs.result }} == "true" ]]
+        then
+          export IMAGE=${{ steps.find-latest-image.outputs.output }}
+        else
+          export IMAGE=${{ inputs.osImage }}
+        fi
+
+        echo "osImage=$IMAGE" >> $GITHUB_OUTPUT
+        echo "Using image: $IMAGE"
+
+        case "$IMAGE" in
+          *"/stream/debug/"*)
+            echo "isDebugImage=true" >> "$GITHUB_OUTPUT"
+            echo "Image is debug image."
+            ;;
+          *)
+            echo "isDebugImage=false" >> "$GITHUB_OUTPUT"
+            echo "Image is not debug image."
+            ;;
+        esac
--- a/.github/workflows/e2e-mini.yml
+++ b/.github/workflows/e2e-mini.yml
@ -1,20 +1,35 @@
 name: e2e test qemu (MiniConstellation)
+# The workflow is triggered by the completion of the release workflow.
+# It is not called by the release pipeline to allow quicker retrying of failed tests
+# The workflow only executes, after being triggered, if the triggering workflow completed successfully.
+# e2e-test-release uses the same branch as the triggering workflow and not the commit of the triggering workflow. This is because the release workflow produces further commits.

 on:
+  workflow_run:
+    workflows: ["Release"]
+    types: [completed]
  workflow_dispatch:
    inputs:
      ref:
        type: string
+        default: ""
        description: "Git ref to checkout"
-        required: false
  workflow_call:
    inputs:
      ref:
        type: string
+        default: ""
        description: "Git ref to checkout"
-        required: true

 jobs:
+  on-failure-quit:
+    runs-on: ubuntu-22.04
+    if: github.event.workflow_run.conclusion == 'failure'
+    steps:
+      - run: |
+          echo 'Release workflow failed, exiting..'
+          exit 1
+
  e2e-mini:
    runs-on: ubuntu-22.04
    environment: e2e
@ -26,7 +41,7 @@ jobs:
        id: checkout
        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
        with:
-          ref: ${{ inputs.ref || github.head_ref }}
+          ref: ${{ inputs.ref || github.event.workflow_run.head_branch || github.head_ref }}

      - name: Azure login OIDC
        uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2 # v1.4.7
--- a/.github/workflows/e2e-test-daily.yml
+++ b/.github/workflows/e2e-test-daily.yml
@ -16,7 +16,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        refStream: ["main-debug", "release-stable"]
+        refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"]
    name: Find latest image
    runs-on: ubuntu-22.04
    permissions:
@ -31,31 +31,20 @@ jobs:
        with:
          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

-      - name: Login to AWS
-        uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0
+      - name: Select relevant image
+        id: select-image-action
+        uses: ./.github/actions/select_image
        with:
-          role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
-          aws-region: eu-central-1
-
-      - name: Separate ref and stream from matrix
-        id: separate-ref-stream
-        env:
-          REFSTREAM: ${{ matrix.refStream }}
-        run: |
-          echo "ref=${REFSTREAM%-*}" | tee -a "$GITHUB_OUTPUT"
-          echo "stream=${REFSTREAM#*-}" | tee -a "$GITHUB_OUTPUT"
-
-      - name: Find latest image
-        id: find-latest-image
-        uses: ./.github/actions/versionsapi
-        with:
-          command: latest
-          ref: ${{ steps.separate-ref-stream.outputs.ref == 'release' && '-' || steps.separate-ref-stream.outputs.ref }}
-          stream: ${{ steps.separate-ref-stream.outputs.stream }}
+          osImage: ${{ matrix.refStream }}

      - name: Relabel output
        id: relabel-output
-        run: echo "image-${{ matrix.refStream }}=${{ steps.find-latest-image.outputs.output }}" | tee -a "$GITHUB_OUTPUT"
+        shell: bash
+        run: |
+          ref=$(echo ${{ matrix.refStream }} | cut -d/ -f2)
+          stream=$(echo ${{ matrix.refStream }} | cut -d/ -f4)
+
+          echo "image-$ref-$stream=${{ steps.select-image-action.outputs.osImage }}" | tee -a "$GITHUB_OUTPUT"

  e2e-daily:
    strategy:
@ -63,7 +52,7 @@ jobs:
      max-parallel: 5
      matrix:
        provider: ["gcp", "azure", "aws"]
-        refStream: ["main-debug", "release-stable"]
+        refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"]
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
@ -105,9 +94,9 @@ jobs:
          workerNodesCount: "2"
          controlNodesCount: "3"
          cloudProvider: ${{ matrix.provider }}
-          osImage: ${{ matrix.refStream == 'release-stable' && needs.find-latest-image.outputs.image-release-stable || needs.find-latest-image.outputs.image-main-debug }}
-          isDebugImage: ${{ matrix.refStream == 'main-debug' }}
-          cliVersion: ${{ matrix.refStream == 'release-stable' && needs.find-latest-image.outputs.image-release-stable || '' }}
+          osImage: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || needs.find-latest-image.outputs.image-main-debug }}
+          isDebugImage: ${{ matrix.refStream == 'ref/main/stream/debug/?' }}
+          cliVersion: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || '' }}
          azureSubscription: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
          azureTenant: ${{ secrets.AZURE_E2E_TENANT_ID }}
          azureClientID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
--- a/.github/workflows/e2e-test-release.yml
+++ b/.github/workflows/e2e-test-release.yml
@ -0,0 +1,218 @@
+name: e2e test release
+# This workflow is not integrated with e2e-test-weekly since we want different tests to run during weekly and release testing.
+# To integrate both tests we would need to pass executed tests as arguments.
+# Defining the executed tests is currently the main point of the e2e-test-weekly workflow.
+# e2e-test-release runs the same tests as e2e-test-weekly except:
+# - any tests on the last release
+# - loadbalancer tests for AWS. Test test is currently broken and should not block a release. AB#2780.
+#
+# The workflow is triggered by the completion of the release workflow.
+# The workflow only executes, after being triggered, if the triggering workflow completed successfully.
+# e2e-test-release uses the same branch as the triggering workflow and not the commit of the triggering workflow. This is because the release workflow produces further commits.
+# e2e-test-release depends on the fact that actions/constellation_create does not overwrite the default osImage, if no osImage is supplied.
+
+on:
+  workflow_dispatch:
+  workflow_run:
+    workflows: ["Release"]
+    types: [completed]
+
+env:
+  ARM_CLIENT_ID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
+  ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+  ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
+  ARM_TENANT_ID: ${{ secrets.AZURE_E2E_TENANT_ID }}
+
+jobs:
+  on-failure-quit:
+    runs-on: ubuntu-22.04
+    if: github.event.workflow_run.conclusion == 'failure'
+    steps:
+      - run: |
+          echo 'Release workflow failed, exiting..'
+          exit 1
+
+  e2e-weekly:
+    needs: [on-failure-quit]
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        test:
+          [
+            "sonobuoy full",
+            "autoscaling",
+            "perf-bench",
+            "lb",
+            "verify",
+            "recover",
+            "iamcreate",
+          ]
+        provider: ["gcp", "azure", "aws"]
+        kubernetes-version: ["v1.24", "v1.25", "v1.26"]
+        runner: [ubuntu-22.04, macos-12]
+        exclude:
+          # IAM create test runs only on latest kubernetes-version.
+          - test: "iamcreate"
+            kubernetes-version: "v1.24"
+          - test: "iamcreate"
+            kubernetes-version: "v1.25"
+          # Verify test runs only on latest kubernetes-version.
+          - test: "verify"
+            kubernetes-version: "v1.24"
+          - test: "verify"
+            kubernetes-version: "v1.25"
+          # Recover test runs only on latest kubernetes-version.
+          - test: "recover"
+            kubernetes-version: "v1.24"
+          - test: "recover"
+            kubernetes-version: "v1.25"
+          # Autoscaling test runs only on latest kubernetes-version.
+          - test: "autoscaling"
+            kubernetes-version: "v1.24"
+          - test: "autoscaling"
+            kubernetes-version: "v1.25"
+          # Perf-Bench test runs only on latest kubernetes-version.
+          - test: "perf-bench"
+            kubernetes-version: "v1.24"
+          - test: "perf-bench"
+            kubernetes-version: "v1.25"
+          # lb test runs only on latest kubernetes-version.
+          - test: "lb"
+            kubernetes-version: "v1.24"
+          - test: "lb"
+            kubernetes-version: "v1.25"
+          # Currently not supported on AWS.
+          - test: "autoscaling"
+            provider: "aws"
+          - test: "perf-bench"
+            provider: "aws"
+          # Currently broken on AWS. Enable when AB#2780 is fixed.
+          - test: "lb"
+            provider: "aws"
+    runs-on: ${{ matrix.runner }}
+    permissions:
+      id-token: write
+      checks: write
+      contents: read
+    steps:
+      - name: Install the basics tools (macOS)
+        if: runner.os == 'macOS'
+        shell: bash
+        run: brew install coreutils kubectl bash
+
+      - name: Checkout
+        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
+        with:
+          fetch-depth: 0
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.event.workflow_run.head_branch || '' }}
+
+      - name: Setup Go environment
+        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version: "1.20.2"
+
+      - name: Set up gcloud CLI (macOS)
+        if: matrix.provider == 'gcp' && runner.os == 'macOS'
+        uses: google-github-actions/setup-gcloud@d51b5346f85640ec2aa2fa057354d2b82c2fcbce # v1.0.1
+
+      - name: Login to Azure
+        if: matrix.provider == 'azure'
+        uses: ./.github/actions/login_azure
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
+
+      - name: Create Azure resource group
+        id: az_resource_group_gen
+        if: matrix.provider == 'azure'
+        shell: bash
+        run: |
+          uuid=$(cat /proc/sys/kernel/random/uuid)
+          name=e2e-test-${uuid%%-*}
+          az group create --location northeurope --name "$name" --tags e2e
+          echo "res_group_name=$name" >> "$GITHUB_OUTPUT"
+
+      - name: Run E2E test
+        id: e2e_test
+        uses: ./.github/actions/e2e_test
+        with:
+          workerNodesCount: "2"
+          controlNodesCount: "3"
+          cloudProvider: ${{ matrix.provider }}
+          cliVersion: ""
+          kubernetesVersion: ${{ matrix.kubernetes-version }}
+          osImage: ""
+          isDebugImage: "false"
+          keepMeasurements: "true"
+          awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
+          awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
+          awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
+          azureSubscription: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
+          azureTenant: ${{ secrets.AZURE_E2E_TENANT_ID }}
+          azureClientID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
+          azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+          azureUserAssignedIdentity: ${{ secrets.AZURE_E2E_USER_ASSIGNED_IDENTITY }}
+          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
+          gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
+          gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
+          gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
+          test: ${{ matrix.test }}
+          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+
+      - name: Always terminate cluster
+        if: always()
+        continue-on-error: true
+        uses: ./.github/actions/constellation_destroy
+        with:
+          kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+
+      - name: Always delete IAM configuration
+        if: always() && matrix.test == 'iamcreate' && matrix.provider != 'azure' # skip for Azure, as the SP / MI does not have the required permissions
+        continue-on-error: true
+        uses: ./.github/actions/constellation_iam_destroy
+
+      - name: Notify teams channel
+        if: failure() && github.ref == 'refs/heads/main'
+        continue-on-error: true
+        shell: bash
+        working-directory: .github/actions/e2e_test
+        run: |
+          sudo apt-get install gettext-base -y
+          export TEAMS_JOB_NAME=${{ matrix.provider }}
+          export TEAMS_RUN_ID=${{ github.run_id }}
+          envsubst < teams-payload.json > to-be-send.json
+          curl                                          \
+            -H "Content-Type: application/json"         \
+            -d @to-be-send.json                         \
+            "${{ secrets.MS_TEAMS_WEBHOOK_URI }}"
+
+      - name: Always destroy Azure resource group
+        if: always() && matrix.provider == 'azure'
+        shell: bash
+        run: |
+          az group delete \
+            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
+            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
+            --force-deletion-types Microsoft.Compute/virtualMachines \
+            --no-wait \
+            --yes
+
+  e2e-upgrade:
+    strategy:
+        fail-fast: false
+        max-parallel: 1
+        matrix:
+          fromVersion:
+            ["v2.6.0"]
+          cloudProvider: ["gcp", "azure"]
+    name: Run upgrade tests
+    secrets: inherit
+    permissions:
+      id-token: write
+      contents: read
+    uses: ./.github/workflows/e2e-upgrade.yml
+    with:
+      fromVersion: ${{ matrix.fromVersion }}
+      cloudProvider: ${{ matrix.cloudProvider }}
+      workerNodesCount: 2
+      controlNodesCount: 3
--- a/.github/workflows/e2e-test-weekly.yml
+++ b/.github/workflows/e2e-test-weekly.yml
@ -16,7 +16,7 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        refStream: ["main-debug", "release-stable"]
+        refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"]
    name: Find latest image
    runs-on: ubuntu-22.04
    permissions:
@ -31,36 +31,25 @@ jobs:
        with:
          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

-      - name: Login to AWS
-        uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0
+      - name: Select relevant image
+        id: select-image-action
+        uses: ./.github/actions/select_image
        with:
-          role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
-          aws-region: eu-central-1
-
-      - name: Separate ref and stream from matrix
-        id: separate-ref-stream
-        env:
-          REFSTREAM: ${{ matrix.refStream }}
-        run: |
-          echo "ref=${REFSTREAM%-*}" | tee -a "$GITHUB_OUTPUT"
-          echo "stream=${REFSTREAM#*-}" | tee -a "$GITHUB_OUTPUT"
-
-      - name: Find latest image
-        id: find-latest-image
-        uses: ./.github/actions/versionsapi
-        with:
-          command: latest
-          ref: ${{ steps.separate-ref-stream.outputs.ref == 'release' && '-' || steps.separate-ref-stream.outputs.ref }}
-          stream: ${{ steps.separate-ref-stream.outputs.stream }}
+          osImage: ${{ matrix.refStream }}

      - name: Relabel output
        id: relabel-output
-        run: echo "image-${{ matrix.refStream }}=${{ steps.find-latest-image.outputs.output }}" | tee -a "$GITHUB_OUTPUT"
+        shell: bash
+        run: |
+          ref=$(echo ${{ matrix.refStream }} | cut -d/ -f2)
+          stream=$(echo ${{ matrix.refStream }} | cut -d/ -f4)
+
+          echo "image-$ref-$stream=${{ steps.select-image-action.outputs.osImage }}" | tee -a "$GITHUB_OUTPUT"

  e2e-weekly:
    strategy:
      fail-fast: false
-      max-parallel: 5
+      max-parallel: 4
      matrix:
        test:
          [
@ -74,7 +63,7 @@ jobs:
          ]
        provider: ["gcp", "azure", "aws"]
        kubernetes-version: ["v1.24", "v1.25", "v1.26"]
-        refStream: ["main-debug", "release-stable"]
+        refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"]
        exclude:
          # IAM create test runs only on latest kubernetes-version.
          - test: "iamcreate"
@ -112,21 +101,21 @@ jobs:
          - test: "perf-bench"
            provider: "aws"
          # Only iamcreate for K8s v1.25 on all providers.
-          - refStream: "release-stable"
+          - refStream: "ref/release/stream/stable/?"
            kubernetes-version: "v1.24"
-          - refStream: "release-stable"
+          - refStream: "ref/release/stream/stable/?"
            kubernetes-version: "v1.26"
-          - refStream: "release-stable"
+          - refStream: "ref/release/stream/stable/?"
            test: "lb"
-          - refStream: "release-stable"
+          - refStream: "ref/release/stream/stable/?"
            test: "perf-bench"
-          - refStream: "release-stable"
+          - refStream: "ref/release/stream/stable/?"
            test: "autoscaling"
-          - refStream: "release-stable"
+          - refStream: "ref/release/stream/stable/?"
            test: "sonobuoy full"
-          - refStream: "release-stable"
+          - refStream: "ref/release/stream/stable/?"
            test: "verify"
-          - refStream: "release-stable"
+          - refStream: "ref/release/stream/stable/?"
            test: "recover"
    runs-on: ubuntu-22.04
    permissions:
@ -169,9 +158,9 @@ jobs:
          workerNodesCount: "2"
          controlNodesCount: "3"
          cloudProvider: ${{ matrix.provider }}
-          osImage: ${{ matrix.refStream == 'release-stable' && needs.find-latest-image.outputs.image-release-stable || needs.find-latest-image.outputs.image-main-debug }}
-          isDebugImage: ${{ matrix.refStream == 'main-debug' }}
-          cliVersion: ${{ matrix.refStream == 'release-stable' && needs.find-latest-image.outputs.image-release-stable || '' }}
+          osImage: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || needs.find-latest-image.outputs.image-main-debug }}
+          isDebugImage: ${{ matrix.refStream == 'ref/main/stream/debug/?' }}
+          cliVersion: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || '' }}
          kubernetesVersion: ${{ matrix.kubernetes-version }}
          awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
          awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
--- a/.github/workflows/e2e-upgrade.yml
+++ b/.github/workflows/e2e-upgrade.yml
@ -20,9 +20,10 @@ on:
        description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
        type: string
        required: true
-      toCLI:
-        description: CLI version to execute upgrade with, e.g., 'v2.1.3', or empty to build HEAD.
+      gitRef:
+        description: Ref to build upgrading CLI on, empty for HEAD.
        type: string
+        default: "head"
        required: false
      toImage:
        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
@ -54,9 +55,10 @@ on:
        description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
        type: string
        required: true
-      toCLI:
-        description: CLI version to execute upgrade with, e.g., 'v2.1.3', or empty to build HEAD.
+      gitRef:
+        description: Ref to build upgrading CLI on.
        type: string
+        default: "head"
        required: false
      toImage:
        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
@ -85,12 +87,20 @@ jobs:
      id-token: write
      contents: read
    steps:
-      - name: Check out repository
-        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v3.2.0
+      - name: Checkout
+        if: inputs.gitRef == 'head'
+        uses: actions/checkout@24cb9080177205b6e8c946b17badbe402adc938f # v3.4.0
        with:
          fetch-depth: 0
          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

+      - name: Checkout ref
+        if: inputs.gitRef != 'head'
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gitRef }}
+
      - name: Setup Go environment
        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
        with:
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@ -203,7 +203,7 @@ jobs:

  update-versions:
    name: Update container image versions
-    needs: [verify-inputs, micro-services]
+    needs: [verify-inputs, micro-services, micro-services-metadata]
    runs-on: ubuntu-22.04
    permissions:
      contents: write
@ -294,45 +294,9 @@ jobs:
          git commit -m "attestation: hardcode measurements for ${VERSION}"
          git push

-  e2e-tests:
-    name: Run E2E tests
-    needs: [verify-inputs, update-hardcoded-measurements]
-    secrets: inherit
-    strategy:
-      matrix:
-        runner: [ubuntu-22.04, macos-12]
-        csp: [aws, azure, gcp]
-    uses: ./.github/workflows/e2e-test-manual.yml
-    permissions:
-      id-token: write
-      contents: read
-      checks: write
-    with:
-      workerNodesCount: 2
-      controlNodesCount: 3
-      cloudProvider: ${{ matrix.csp }}
-      runner: ${{ matrix.runner }}
-      test: "sonobuoy full"
-      kubernetesVersion: "v1.25"
-      keepMeasurements: true
-      cliImageVersion: ":${{ inputs.version }}"
-      machineType: "default"
-      git-ref: ${{ needs.verify-inputs.outputs.RELEASE_BRANCH }}
-
-  e2e-mini:
-    name: Run E2E tests for mini Constellation
-    needs: [verify-inputs, update-hardcoded-measurements]
-    uses: ./.github/workflows/e2e-mini.yml
-    permissions:
-      id-token: write
-      contents: read
-    secrets: inherit
-    with:
-      ref: ${{ needs.verify-inputs.outputs.RELEASE_BRANCH }}
-
  tag-release:
    name: Tag release
-    needs: [verify-inputs, e2e-tests, e2e-mini]
+    needs: [verify-inputs, update-hardcoded-measurements]
    runs-on: ubuntu-22.04
    permissions:
      contents: write