ci: parallelize upgrade e2e test

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

.github/actions/artifact_download/action.yml

@@ -9,7 +9,7 @@ inputs:
     description: 'Download to a specified path.'
     required: false
     default: ./
-  encryptionSecret:
+  encryption-secret:
     description: 'The secret to use for decrypting the artifact.'
     required: true
 
@@ -37,4 +37,4 @@ runs:
       shell: bash
       run: |
         mkdir -p ${{ inputs.path }}
-        unzip -P '${{ inputs.encryptionSecret }}' -qq -d ${{ inputs.path }} ${{ steps.tempdir.outputs.directory }}/archive.zip
+        unzip -P '${{ inputs.encryption-secret }}' -qq -d ${{ inputs.path }} ${{ steps.tempdir.outputs.directory }}/archive.zip

.github/actions/artifact_upload/action.yml

@@ -11,7 +11,7 @@ inputs:
   retention-days:
     description: 'How long the artifact should be retained for.'
     default: 60
-  encryptionSecret:
+  encryption-secret:
     description: 'The secret to use for encrypting the files.'
     required: true
 
@@ -58,7 +58,7 @@ runs:
         for target in ${paths}
         do
           pushd "$(dirname "${target}")" || exit 1
-          zip -e -P '${{ inputs.encryptionSecret }}' -r "${{ steps.tempdir.outputs.directory }}/archive.zip" "$(basename "${target}")"
+          zip -e -P '${{ inputs.encryption-secret }}' -r "${{ steps.tempdir.outputs.directory }}/archive.zip" "$(basename "${target}")"
           popd || exit 1
         done
 

.github/actions/constellation_create/action.yml

@@ -59,7 +59,7 @@ inputs:
   force:
     description: "Set the force-flag on apply to ignore version mismatches."
     required: false
-  encryptionSecret:
+  encryption-secret:
     description: "The secret to use for encrypting the artifact."
     required: true
 
@@ -274,4 +274,4 @@ runs:
         name: serial-logs-${{ inputs.artifactNameSuffix }}
         path: >
           !(terraform).log
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}

.github/actions/e2e_benchmark/action.yml

@@ -17,7 +17,7 @@ inputs:
   awsOpenSearchPwd:
     description: "AWS OpenSearch Password to upload the results."
     required: false
-  encryptionSecret:
+  encryption-secret:
     description: 'The secret to use for encrypting the artifact.'
     required: true
 
@@ -100,7 +100,7 @@ runs:
       with:
         path: "out/fio-constellation-${{ inputs.cloudProvider }}.json"
         name: "fio-constellation-${{ inputs.cloudProvider }}.json"
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}
 
     - name: Run knb benchmark
       shell: bash
@@ -122,7 +122,7 @@ runs:
       with:
         path: "out/knb-constellation-${{ inputs.cloudProvider }}.json"
         name: "knb-constellation-${{ inputs.cloudProvider }}.json"
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}
 
     - name: Assume AWS role to retrieve and update benchmarks in S3
       uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
@@ -176,7 +176,7 @@ runs:
         path: >
           benchmarks/constellation-${{ inputs.cloudProvider }}.json
         name: "benchmarks"
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}
 
     - name: Upload benchmark results to opensearch
       if: (!env.ACT)

.github/actions/e2e_sonobuoy/action.yml

@@ -11,7 +11,7 @@ inputs:
   kubeconfig:
     description: "The kubeconfig of the cluster to test."
     required: true
-  encryptionSecret:
+  encryption-secret:
     description: 'The secret to use for encrypting the artifact.'
     required: true
 
@@ -54,7 +54,7 @@ runs:
       with:
         name: "sonobuoy-logs-${{ inputs.artifactNameSuffix }}.tar.gz"
         path: "*_sonobuoy_*.tar.gz"
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}
 
     # Only works on "sonobuoy full" tests (e2e plugin)
     - name: Extract test results

.github/actions/e2e_test/action.yml

@@ -86,7 +86,7 @@ inputs:
   force:
     description: "Set the force-flag on apply to ignore version mismatches."
     required: false
-  encryptionSecret:
+  encryption-secret:
     description: 'The secret to use for decrypting the artifact.'
     required: true
 
@@ -299,7 +299,7 @@ runs:
         clusterCreation: ${{ inputs.clusterCreation }}
         marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
         force: ${{ inputs.force }}
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}
 
     - name: Deploy log- and metrics-collection (Kubernetes)
       id: deploy-logcollection
@@ -334,7 +334,7 @@ runs:
         sonobuoyTestSuiteCmd: "--mode quick"
         kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
         artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}
 
     - name: Run sonobuoy full test
       if: inputs.test == 'sonobuoy full'
@@ -344,7 +344,7 @@ runs:
         sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol|Services should serve endpoints on same port and different protocols" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
         kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
         artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}
 
     - name: Run autoscaling test
       if: inputs.test == 'autoscaling'
@@ -367,7 +367,7 @@ runs:
         awsOpenSearchDomain: ${{ inputs.awsOpenSearchDomain }}
         awsOpenSearchUsers: ${{ inputs.awsOpenSearchUsers }}
         awsOpenSearchPwd: ${{ inputs.awsOpenSearchPwd }}
-        encryptionSecret: ${{ inputs.encryptionSecret }}
+        encryption-secret: ${{ inputs.encryption-secret }}
 
     - name: Run constellation verify test
       if: inputs.test == 'verify'

.github/workflows/e2e-test-daily.yml

@@ -91,7 +91,7 @@ jobs:
           awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
           awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
           clusterCreation: "cli"
-          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 
       - name: Always terminate cluster
         if: always()

.github/workflows/e2e-test-release.yml

@@ -265,7 +265,7 @@ jobs:
           clusterCreation: ${{ matrix.clusterCreation }}
           s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }}
           s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }}
-          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 
       - name: Always terminate cluster
         if: always()

.github/workflows/e2e-test-weekly.yml

@@ -305,7 +305,7 @@ jobs:
           clusterCreation: ${{ matrix.clusterCreation }}
           s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }}
           s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }}
-          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 
       - name: Always terminate cluster
         if: always()

.github/workflows/e2e-test.yml

@@ -246,7 +246,7 @@ jobs:
           s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }}
           marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
           force: ${{ inputs.force }}
-          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 
       - name: Always terminate cluster
         if: always()

.github/workflows/e2e-upgrade.yml

@@ -111,7 +111,57 @@ jobs:
           echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT"
           echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT"
 
-  e2e-upgrade:
+  build-target-cli:
+    name: Build upgrade target version CLI
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      checks: write
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        if: inputs.gitRef == 'head'
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          fetch-depth: 0
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
+
+      - name: Checkout ref
+        if: inputs.gitRef != 'head'
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gitRef }}
+
+      - name: Setup Bazel & Nix
+        uses: ./.github/actions/setup_bazel_nix
+        with:
+          useCache: "true"
+          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+
+      - name: Log in to the Container registry
+        uses: ./.github/actions/container_registry_login
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build CLI
+        uses: ./.github/actions/build_cli
+        with:
+          enterpriseCLI: true
+          outputPath: "build/constellation"
+          push: true
+
+      - name: Upload CLI binary
+        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        with:
+          name: constellation
+          path: build/constellation
+
+  create-cluster:
+    name: Create upgrade origin version cluster
     runs-on: ubuntu-22.04
     permissions:
       id-token: write
@@ -119,6 +169,8 @@ jobs:
       contents: read
       packages: write
     needs: [split-nodeCount]
+    outputs:
+      kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
     steps:
       - name: Checkout
         if: inputs.gitRef == 'head'
@@ -139,26 +191,6 @@ jobs:
           useCache: "true"
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
 
-      - name: Login to AWS
-        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
-        with:
-          role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
-          aws-region: eu-central-1
-
-      - name: Find latest nightly image
-        id: find-image
-        if: inputs.toImage == ''
-        uses: ./.github/actions/versionsapi
-        with:
-          command: latest
-          ref: main
-          stream: nightly
-
-      - name: Simulate patch upgrade
-        if: inputs.simulatedTargetVersion != ''
-        run: |
-          echo ${{ inputs.simulatedTargetVersion }} > version.txt
-
       - name: Create cluster with 'fromVersion' CLI.
         id: e2e_test
         uses: ./.github/actions/e2e_test
@@ -183,14 +215,80 @@ jobs:
           awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
           awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
           clusterCreation: "cli"
-          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 
-      - name: Build CLI
-        uses: ./.github/actions/build_cli
+      - name: Remove Terraform plugin cache
+        if: always()
+        run: |
+          rm -rf constellation-terraform/.terraform
+          rm -rf constellation-iam-terraform/.terraform
+
+      - name: Upload Working Directory
+        if: always()
+        uses: ./.github/actions/artifact_upload
         with:
-          enterpriseCLI: true
-          outputPath: "build/constellation"
-          push: true
+          name: constellation-pre-test
+          path: >
+            ${{ steps.e2e_test.outputs.kubeconfig }}
+            constellation-terraform
+            constellation-iam-terraform
+            constellation-conf.yaml
+            constellation-state.yaml
+            constellation-mastersecret.json
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+
+  e2e-upgrade:
+    name: Run upgrade test
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      checks: write
+      contents: read
+      packages: write
+    needs:
+      - split-nodeCount
+      - build-target-cli
+      - create-cluster
+    steps:
+      - name: Checkout
+        if: inputs.gitRef == 'head'
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          fetch-depth: 0
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
+
+      - name: Checkout ref
+        if: inputs.gitRef != 'head'
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gitRef }}
+
+      - name: Setup Bazel & Nix
+        uses: ./.github/actions/setup_bazel_nix
+        with:
+          useCache: "true"
+          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+
+      - name: Login to AWS
+        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
+          aws-region: eu-central-1
+
+      - name: Find latest nightly image
+        id: find-image
+        if: inputs.toImage == ''
+        uses: ./.github/actions/versionsapi
+        with:
+          command: latest
+          ref: main
+          stream: nightly
+
+      - name: Simulate patch upgrade
+        if: inputs.simulatedTargetVersion != ''
+        run: |
+          echo ${{ inputs.simulatedTargetVersion }} > version.txt
 
       - name: Login to GCP (IAM service account)
         if: inputs.cloudProvider == 'gcp'
@@ -213,6 +311,25 @@ jobs:
         with:
           azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
 
+      - name: Download CLI
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: constellation
+          path: build
+
+      - name: Download Working Directory (Pre-test)
+        uses: ./.github/actions/artifact_download
+        with:
+          name: constellation-pre-test
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          
+      - name: Make Constellation executable and add to PATH
+        if: always()
+        run: |
+          chmod +x build/constellation
+          export PATH="$PATH:build"
+          echo "build" >> "$GITHUB_PATH"
+
       - name: Migrate config
         id: constellation-config-migrate
         run: |
@@ -245,7 +362,7 @@ jobs:
 
       - name: Run upgrade test
         env:
-          KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }}
+          KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }}
           IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }}
           KUBERNETES: ${{ inputs.toKubernetes }}
           MICROSERVICES: ${{ inputs.toMicroservices }}
@@ -265,10 +382,82 @@ jobs:
 
           bazel run //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG"
 
+      - name: Remove Terraform plugin cache
+        if: always()
+        run: |
+          rm -rf constellation-terraform/.terraform
+          rm -rf constellation-iam-terraform/.terraform
+
+      - name: Upload Working Directory
+        if: always()
+        uses: ./.github/actions/artifact_upload
+        with:
+          name: constellation-post-test
+          path: |
+            ${{ needs.create-cluster.outputs.kubeconfig }}
+            constellation-terraform
+            constellation-iam-terraform
+            constellation-conf.yaml
+            constellation-state.yaml
+            constellation-mastersecret.json
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+
+  clean-up:
+    name: Clean up resources
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      checks: write
+      contents: read
+      packages: write
+    if: always()
+    needs: [create-cluster, e2e-upgrade]
+    steps:
+      - name: Checkout
+        if: inputs.gitRef == 'head'
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          fetch-depth: 0
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
+
+      - name: Checkout ref
+        if: inputs.gitRef != 'head'
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          fetch-depth: 0
+          ref: ${{ inputs.gitRef }}
+
+      - name: Download CLI
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: constellation
+          path: build
+
+      - name: Download Working Directory (Pre-test)
+        if: always() && needs.e2e-upgrade.result != 'success'
+        uses: ./.github/actions/artifact_download
+        with:
+          name: constellation-pre-test
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+
+      - name: Download Working Directory (Post-test)
+        if: always() && needs.e2e-upgrade.result == 'success'
+        uses: ./.github/actions/artifact_download
+        with:
+          name: constellation-post-test
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+
+      - name: Make Constellation executable and add to PATH
+        if: always()
+        run: |
+          chmod +x build/constellation
+          export PATH="$PATH:build"
+          echo "build" >> "$GITHUB_PATH"
+
       - name: Always fetch logs
         if: always()
         env:
-          KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }}
+          KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }}
         run: |
           kubectl logs -n kube-system -l "app.kubernetes.io/name=constellation-operator" --tail=-1 > node-operator.logs
           kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs
@@ -283,14 +472,14 @@ jobs:
             node-operator.logs
             node-maintenance-operator.logs
             constellation-version.yaml
-          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy
         with:
-          kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
           clusterCreation: "cli"
+          kubeconfig: ${{ needs.create-cluster.outputs.kubeconfig }}
           cloudProvider: ${{ inputs.cloudProvider }}
           azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
           gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"
@@ -305,7 +494,9 @@ jobs:
 
       - name: Notify about failure
         if: |
-          failure() &&
+          always() &&
+          needs.create-cluster.result != 'success' &&
+          needs.e2e-upgrade.result != 'success' &&
           github.ref == 'refs/heads/main' &&
           inputs.scheduled
         continue-on-error: true