ci: separate e2e permissions (#1555)

* split e2e test iam create / create perms

* remove global Azure credentials

* remove unnecessary azure actions

* use UUID

* fix e2e upgrade test

* rename create inputs

* remove continue-on-error for resource deletion

* de-exclude verify test

* fix exclude

* fix release e2e test

---------

Co-authored-by: Nils Hanke <nils.hanke@outlook.com>

.github/actions/constellation_create/action.yml

@@ -35,7 +35,7 @@ inputs:
   gcpProject:
     description: "The GCP project to deploy Constellation in."
     required: false
-  gcpClusterServiceAccountKey:
+  gcpInClusterServiceAccountKey:
     description: "The GCP Service account to use inside the created Constellation cluster."
     required: false
   #
@@ -162,7 +162,7 @@ runs:
       if: inputs.cloudProvider == 'gcp' && !inputs.existingConfig # Skip if using existing config. serviceAccountKey.json is already present in that case.
       shell: bash
       env:
-        GCP_CLUSTER_SERVICE_ACCOUNT_KEY: ${{ inputs.gcpClusterServiceAccountKey }}
+        GCP_CLUSTER_SERVICE_ACCOUNT_KEY: ${{ inputs.gcpInClusterServiceAccountKey }}
       run: |
         echo "$GCP_CLUSTER_SERVICE_ACCOUNT_KEY" > serviceAccountKey.json
 

.github/actions/constellation_iam_destroy/action.yml

@@ -1,9 +1,41 @@
 name: Delete IAM configuration
 description: Delete previously created IAM configuration.
 
+inputs:
+  cloudProvider:
+      description: "Either 'aws', 'azure' or 'gcp'."
+      required: true
+  gcpServiceAccount:
+      description: "GCP service account to use for authentication."
+      required: false
+  azureCredentials:
+      description: "Azure service principal to use for authentication."
+      required: false
+
 runs:
   using: "composite"
   steps:
+    - name: Login to GCP (IAM service account)
+      if: inputs.cloudProvider == 'gcp'
+      uses: ./.github/actions/login_gcp
+      with:
+        service_account: ${{ inputs.gcpServiceAccount }}
+
+    - name: Login to AWS (IAM role)
+      if: inputs.cloudProvider == 'aws'
+      uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
+        aws-region: eu-central-1
+        # extend token expiry to 6 hours to ensure constellation can terminate
+        role-duration-seconds: 21600
+
+    - name: Login to Azure (IAM service principal)
+      if: inputs.cloudProvider == 'azure'
+      uses: ./.github/actions/login_azure
+      with:
+        azure_credentials: ${{ inputs.azureCredentials }}
+
     - name: Delete IAM configuration
       shell: bash
       run: |

.github/actions/e2e_test/action.yml

@@ -35,10 +35,13 @@ inputs:
   gcpProject:
     description: "The GCP project to deploy Constellation in."
     required: false
-  gcp_service_account:
-    description: "Service account with permissions to create Constellation on GCP."
+  gcpIAMCreateServiceAccount:
+    description: "Service account with permissions to create IAM configuration on GCP."
     required: false
-  gcpClusterServiceAccountKey:
+  gcpClusterCreateServiceAccount:
+    description: "Service account with permissions to create a Constellation cluster on GCP."
+    required: false
+  gcpInClusterServiceAccountKey:
     description: "Service account to use inside the created Constellation cluster on GCP."
     required: false
   awsOpenSearchDomain:
@@ -50,26 +53,14 @@ inputs:
   awsOpenSearchPwd:
     description: "AWS OpenSearch Password to upload the benchmark results."
     required: false
-  azureSubscription:
-    description: "The Azure subscription ID to deploy Constellation in."
-    required: false
-  azureTenant:
-    description: "The Azure tenant ID to deploy Constellation in."
-    required: false
-  azureClientID:
-    description: "The client ID of the application registration created for Constellation in Azure."
-    required: false
-  azureClientSecret:
-    description: "The client secret value of the used secret"
-    required: false
-  azureUserAssignedIdentity:
-    description: "The Azure user assigned identity to use for Constellation."
-    required: false
-  azureResourceGroup:
-    description: "The resource group to use"
-    required: false
+  azureClusterCreateCredentials:
+    description: "Azure credentials authorized to create a Constellation cluster."
+    required: true
+  azureIAMCreateCredentials:
+    description: "Azure credentials authorized to create an IAM configuration."
+    required: true
   test:
-    description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, nop, iamcreate]."
+    description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, nop]."
     required: true
   sonobuoyTestSuiteCmd:
     description: "The sonobuoy test suite to run."
@@ -87,7 +78,7 @@ runs:
   using: "composite"
   steps:
     - name: Check input
-      if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "nop", "iamcreate"]'), inputs.test))
+      if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "nop"]'), inputs.test))
       shell: bash
       run: |
         echo "::error::Invalid input for test field: ${{ inputs.test }}"
@@ -155,43 +146,75 @@ runs:
         targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
         targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
 
-    - name: Login to GCP
+    - name: Login to GCP (IAM service account)
       if: inputs.cloudProvider == 'gcp'
       uses: ./.github/actions/login_gcp
       with:
-        service_account: ${{ inputs.gcp_service_account }}
+        service_account: ${{ inputs.gcpIAMCreateServiceAccount }}
 
-    - name: Login to AWS
+    - name: Login to AWS (IAM role)
       if: inputs.cloudProvider == 'aws'
       uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0
       with:
-        role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2E
+        role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
         aws-region: eu-central-1
         # extend token expiry to 6 hours to ensure constellation can terminate
         role-duration-seconds: 21600
 
+    - name: Login to Azure (IAM service principal)
+      if: inputs.cloudProvider == 'azure'
+      uses: ./.github/actions/login_azure
+      with:
+        azure_credentials: ${{ inputs.azureIAMCreateCredentials }}
+
+    - name: Create UUID
+      id: create-uuid
+      shell: bash
+      run: |
+        uuid=$(uuidgen)
+        echo "uuid=${uuid%%-*}" >> $GITHUB_OUTPUT
+
     - name: Create IAM configuration
       id: constellation-iam-create
-      if: inputs.test == 'iamcreate' && inputs.cloudProvider != 'azure' # skip for Azure, as the SP / MI does not have the required permissions
       uses: ./.github/actions/constellation_iam_create
       with:
         cloudProvider: ${{ inputs.cloudProvider }}
         awsZone: eu-central-1c
-        awsPrefix: e2e_${{ github.run_id }}_${{ github.run_attempt }}
+        awsPrefix: e2e_${{ github.run_id }}_${{ github.run_attempt }}_${{ steps.create-uuid.outputs.uuid }}
         azureRegion: northeurope
-        azureResourceGroup: e2e_${{ github.run_id }}_${{ github.run_attempt }}_rg
-        azureServicePrincipal: e2e_${{ github.run_id }}_${{ github.run_attempt }}_sp
+        azureResourceGroup: e2e_${{ github.run_id }}_${{ github.run_attempt }}_${{ steps.create-uuid.outputs.uuid }}_rg
+        azureServicePrincipal: e2e_${{ github.run_id }}_${{ github.run_attempt }}_${{ steps.create-uuid.outputs.uuid }}_sp
         gcpProjectID: ${{ inputs.gcpProject }}
         gcpZone: europe-west3-b
-        gcpServiceAccountID: e2e-${{ github.run_id }}-${{ github.run_attempt }}-sa
+        gcpServiceAccountID: e2e-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.create-uuid.outputs.uuid }}-sa
+
+    - name: Login to GCP (Cluster service account)
+      if: inputs.cloudProvider == 'gcp'
+      uses: ./.github/actions/login_gcp
+      with:
+        service_account: ${{ inputs.gcpClusterCreateServiceAccount }}
+
+    - name: Login to AWS (Cluster role)
+      if: inputs.cloudProvider == 'aws'
+      uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster
+        aws-region: eu-central-1
+        # extend token expiry to 6 hours to ensure constellation can terminate
+        role-duration-seconds: 21600
+
+    - name: Login to Azure (Cluster service principal)
+      if: inputs.cloudProvider == 'azure'
+      uses: ./.github/actions/login_azure
+      with:
+        azure_credentials: ${{ inputs.azureClusterCreateCredentials }}
 
     - name: Create cluster
       id: constellation-create
       uses: ./.github/actions/constellation_create
       with:
         cloudProvider: ${{ inputs.cloudProvider }}
-        gcpProject: ${{ inputs.gcpProject }}
-        gcpClusterServiceAccountKey: ${{ inputs.gcpClusterServiceAccountKey }}
+        gcpInClusterServiceAccountKey: ${{ inputs.gcpInClusterServiceAccountKey }}
         workerNodesCount: ${{ inputs.workerNodesCount }}
         controlNodesCount: ${{ inputs.controlNodesCount }}
         machineType: ${{ inputs.machineType }}
@@ -199,12 +222,6 @@ runs:
         isDebugImage: ${{ inputs.isDebugImage }}
         kubernetesVersion: ${{ inputs.kubernetesVersion }}
         keepMeasurements: ${{ inputs.keepMeasurements }}
-        azureSubscription: ${{ inputs.azureSubscription }}
-        azureTenant: ${{ inputs.azureTenant }}
-        azureClientID: ${{ inputs.azureClientID }}
-        azureClientSecret: ${{ inputs.azureClientSecret }}
-        azureUserAssignedIdentity: ${{ inputs.azureUserAssignedIdentity }}
-        azureResourceGroup: ${{ inputs.azureResourceGroup }}
         existingConfig: ${{ steps.constellation-iam-create.outputs.existingConfig }}
 
     #

.github/workflows/e2e-test-daily.yml

@@ -5,12 +5,6 @@ on:
   schedule:
     - cron: "0 3 * * 2-5" # At 03:00 on every day-of-week from Tuesday through Friday.
 
-env:
-  ARM_CLIENT_ID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
-  ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
-  ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-  ARM_TENANT_ID: ${{ secrets.AZURE_E2E_TENANT_ID }}
-
 jobs:
   find-latest-image:
     strategy:
@@ -71,22 +65,6 @@ jobs:
         with:
           go-version: "1.20.3"
 
-      - name: Login to Azure
-        if: matrix.provider == 'azure'
-        uses: ./.github/actions/login_azure
-        with:
-          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
-
-      - name: Create Azure resource group
-        if: matrix.provider == 'azure'
-        id: az_resource_group_gen
-        shell: bash
-        run: |
-          uuid=$(cat /proc/sys/kernel/random/uuid)
-          name=e2e-test-${uuid%%-*}
-          az group create --location northeurope --name "$name" --tags e2e
-          echo "res_group_name=$name" >> "$GITHUB_OUTPUT"
-
       - name: Run E2E test
         id: e2e_test
         uses: ./.github/actions/e2e_test
@@ -97,25 +75,29 @@ jobs:
           osImage: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || needs.find-latest-image.outputs.image-main-debug }}
           isDebugImage: ${{ matrix.refStream == 'ref/main/stream/debug/?' }}
           cliVersion: ${{ matrix.refStream == 'ref/release/stream/stable/?' && needs.find-latest-image.outputs.image-release-stable || '' }}
-          azureSubscription: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-          azureTenant: ${{ secrets.AZURE_E2E_TENANT_ID }}
-          azureClientID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
-          azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
-          azureUserAssignedIdentity: ${{ secrets.AZURE_E2E_USER_ASSIGNED_IDENTITY }}
-          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
-          gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
-          gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
+          gcpClusterCreateServiceAccount: "constellation-e2e-cluster@constellation-331613.iam.gserviceaccount.com"
+          gcpIAMCreateServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+          gcpInClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
           test: "sonobuoy full"
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
+          azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
 
       - name: Always terminate cluster
         if: always()
-        continue-on-error: true
         uses: ./.github/actions/constellation_destroy
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
 
+      - name: Always delete IAM configuration
+        if: always()
+        uses: ./.github/actions/constellation_iam_destroy
+        with:
+          cloudProvider: ${{ matrix.provider }}
+          azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
+          gcpServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+
       - name: Notify teams channel
         if: failure() && github.ref == 'refs/heads/main'
         continue-on-error: true
@@ -130,14 +112,3 @@ jobs:
             -H "Content-Type: application/json"         \
             -d @to-be-send.json                         \
             "${{ secrets.MS_TEAMS_WEBHOOK_URI }}"
-
-      - name: Always destroy Azure resource group
-        if: always() && matrix.provider == 'azure'
-        shell: bash
-        run: |
-          az group delete \
-            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
-            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
-            --force-deletion-types Microsoft.Compute/virtualMachines \
-            --no-wait \
-            --yes

.github/workflows/e2e-test-manual.yml

@@ -108,12 +108,6 @@ on:
         type: string
         required: true
 
-env:
-  ARM_CLIENT_ID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
-  ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
-  ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-  ARM_TENANT_ID: ${{ secrets.AZURE_E2E_TENANT_ID }}
-
 jobs:
   split-cliImageVersion:
     name: Split cliImageVersion
@@ -244,22 +238,6 @@ jobs:
         if: inputs.cloudProvider == 'gcp' && runner.os == 'macOS'
         uses: google-github-actions/setup-gcloud@62d4898025f6041e16b1068643bfc5a696863587 # v1.1.0
 
-      - name: Login to Azure
-        if: inputs.cloudProvider == 'azure'
-        uses: ./.github/actions/login_azure
-        with:
-          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
-
-      - name: Create Azure resource group
-        id: az_resource_group_gen
-        if: inputs.cloudProvider == 'azure'
-        shell: bash
-        run: |
-          uuid=$(uuidgen)
-          name=e2e-test-${uuid%%-*}
-          az group create --location westus --name "$name" --tags e2e
-          echo "res_group_name=$name" >> "$GITHUB_OUTPUT"
-
       - name: Run manual E2E test
         id: e2e_test
         uses: ./.github/actions/e2e_test
@@ -269,24 +247,21 @@ jobs:
           cloudProvider: ${{ inputs.cloudProvider }}
           machineType: ${{ inputs.machineType }}
           gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
-          gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
-          gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
+          gcpClusterCreateServiceAccount: "constellation-e2e-cluster@constellation-331613.iam.gserviceaccount.com"
+          gcpIAMCreateServiceAccount: " constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+          gcpInClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
           test: ${{ inputs.test }}
           kubernetesVersion: ${{ inputs.kubernetesVersion }}
           keepMeasurements: ${{ inputs.keepMeasurements }}
           awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
           awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
           awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
-          azureSubscription: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-          azureTenant: ${{ secrets.AZURE_E2E_TENANT_ID }}
-          azureClientID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
-          azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
-          azureUserAssignedIdentity: ${{ secrets.AZURE_E2E_USER_ASSIGNED_IDENTITY }}
-          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           osImage: ${{ needs.find-latest-image.outputs.image }}
           cliVersion: ${{ needs.split-cliImageVersion.outputs.cliVersion }}
           isDebugImage: ${{ needs.find-latest-image.outputs.isDebugImage }}
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
+          azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
 
       - name: Always terminate cluster
         if: always()
@@ -295,16 +270,9 @@ jobs:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
 
       - name: Always delete IAM configuration
-        if: always() && inputs.test == 'iamcreate' && inputs.cloudProvider != 'azure' # skip for Azure, as the SP / MI does not have the required permissions
+        if: always()
         uses: ./.github/actions/constellation_iam_destroy
-
-      - name: Always destroy Azure resource group
-        if: always() && inputs.cloudProvider == 'azure'
-        shell: bash
-        run: |
-          az group delete \
-            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
-            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
-            --force-deletion-types Microsoft.Compute/virtualMachines \
-            --no-wait \
-            --yes
+        with:
+          cloudProvider: ${{ inputs.cloudProvider }}
+          azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
+          gcpServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"

.github/workflows/e2e-test-release.yml

@@ -150,23 +150,21 @@ jobs:
           cloudProvider: ${{ matrix.provider }}
           cliVersion: ""
           kubernetesVersion: ${{ matrix.kubernetes-version }}
+          keepMeasurements: "true"
           osImage: ""
           isDebugImage: "false"
-          keepMeasurements: "true"
           awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
           awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
           awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
-          azureSubscription: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-          azureTenant: ${{ secrets.AZURE_E2E_TENANT_ID }}
-          azureClientID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
-          azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
-          azureUserAssignedIdentity: ${{ secrets.AZURE_E2E_USER_ASSIGNED_IDENTITY }}
-          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
-          gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
-          gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
+          gcpClusterCreateServiceAccount: "constellation-e2e-cluster@constellation-331613.iam.gserviceaccount.com"
+          gcpIAMCreateServiceAccount: " constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+          gcpInClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
           test: ${{ matrix.test }}
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
+          azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
+
 
       - name: Always terminate cluster
         if: always()
@@ -176,9 +174,13 @@ jobs:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
 
       - name: Always delete IAM configuration
-        if: always() && matrix.test == 'iamcreate' && matrix.provider != 'azure' # skip for Azure, as the SP / MI does not have the required permissions
+        if: always()
         continue-on-error: true
         uses: ./.github/actions/constellation_iam_destroy
+        with:
+          cloudProvider: ${{ matrix.provider }}
+          azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
+          gcpServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
 
       - name: Notify teams channel
         if: failure() && github.ref == 'refs/heads/main'

.github/workflows/e2e-test-weekly.yml

@@ -5,12 +5,6 @@ on:
   schedule:
     - cron: "0 3 * * 6" # At 03:00 on Saturday.
 
-env:
-  ARM_CLIENT_ID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
-  ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
-  ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-  ARM_TENANT_ID: ${{ secrets.AZURE_E2E_TENANT_ID }}
-
 jobs:
   find-latest-image:
     strategy:
@@ -59,21 +53,17 @@ jobs:
             "lb",
             "verify",
             "recover",
-            "iamcreate",
           ]
         provider: ["gcp", "azure", "aws"]
         kubernetes-version: ["v1.24", "v1.25", "v1.26"]
         refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"]
         exclude:
-          # IAM create test runs only on latest kubernetes-version.
-          - test: "iamcreate"
-            kubernetes-version: "v1.24"
-          - test: "iamcreate"
-            kubernetes-version: "v1.25"
-          # Verify test runs only on latest kubernetes-version.
+          # Verify test runs only on latest kubernetes-version by default.
+          # Exclude it for main-debug as it needs to run on 1.25 / release-stable.
           - test: "verify"
             kubernetes-version: "v1.24"
-          - test: "verify"
+          - refStream: "ref/main/stream/debug/?"
+            test: "verify"
             kubernetes-version: "v1.25"
           # Recover test runs only on latest kubernetes-version.
           - test: "recover"
@@ -100,7 +90,7 @@ jobs:
             provider: "aws"
           - test: "perf-bench"
             provider: "aws"
-          # Only iamcreate for K8s v1.25 on all providers.
+          # Only verify for K8s v1.25 on all providers.
           - refStream: "ref/release/stream/stable/?"
             kubernetes-version: "v1.24"
           - refStream: "ref/release/stream/stable/?"
@@ -113,8 +103,6 @@ jobs:
             test: "autoscaling"
           - refStream: "ref/release/stream/stable/?"
             test: "sonobuoy full"
-          - refStream: "ref/release/stream/stable/?"
-            test: "verify"
           - refStream: "ref/release/stream/stable/?"
             test: "recover"
     runs-on: ubuntu-22.04
@@ -135,22 +123,6 @@ jobs:
         with:
           go-version: "1.20.3"
 
-      - name: Login to Azure
-        if: matrix.provider == 'azure'
-        uses: ./.github/actions/login_azure
-        with:
-          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
-
-      - name: Create Azure resource group
-        id: az_resource_group_gen
-        if: matrix.provider == 'azure'
-        shell: bash
-        run: |
-          uuid=$(cat /proc/sys/kernel/random/uuid)
-          name=e2e-test-${uuid%%-*}
-          az group create --location northeurope --name "$name" --tags e2e
-          echo "res_group_name=$name" >> "$GITHUB_OUTPUT"
-
       - name: Run E2E test
         id: e2e_test
         uses: ./.github/actions/e2e_test
@@ -165,29 +137,28 @@ jobs:
           awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
           awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
           awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
-          azureSubscription: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-          azureTenant: ${{ secrets.AZURE_E2E_TENANT_ID }}
-          azureClientID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
-          azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
-          azureUserAssignedIdentity: ${{ secrets.AZURE_E2E_USER_ASSIGNED_IDENTITY }}
-          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
-          gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
-          gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
+          gcpClusterCreateServiceAccount: "constellation-e2e-cluster@constellation-331613.iam.gserviceaccount.com"
+          gcpIAMCreateServiceAccount: " constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+          gcpInClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
           test: ${{ matrix.test }}
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
+          azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
 
       - name: Always terminate cluster
         if: always()
-        continue-on-error: true
         uses: ./.github/actions/constellation_destroy
         with:
           kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
 
       - name: Always delete IAM configuration
-        if: always() && matrix.test == 'iamcreate' && matrix.provider != 'azure' # skip for Azure, as the SP / MI does not have the required permissions
-        continue-on-error: true
+        if: always()
         uses: ./.github/actions/constellation_iam_destroy
+        with:
+          cloudProvider: ${{ matrix.provider }}
+          azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
+          gcpServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
 
       - name: Notify teams channel
         if: failure() && github.ref == 'refs/heads/main'
@@ -204,17 +175,6 @@ jobs:
             -d @to-be-send.json                         \
             "${{ secrets.MS_TEAMS_WEBHOOK_URI }}"
 
-      - name: Always destroy Azure resource group
-        if: always() && matrix.provider == 'azure'
-        shell: bash
-        run: |
-          az group delete \
-            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
-            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
-            --force-deletion-types Microsoft.Compute/virtualMachines \
-            --no-wait \
-            --yes
-
   e2e-upgrade:
     strategy:
         fail-fast: false

.github/workflows/e2e-upgrade.yml

@@ -147,17 +147,14 @@ jobs:
           osImage: ${{ inputs.fromVersion }}
           cliVersion: ${{ inputs.fromVersion }}
           isDebugImage: "false"
-          azureSubscription: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
-          azureTenant: ${{ secrets.AZURE_E2E_TENANT_ID }}
-          azureClientID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
-          azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
-          azureUserAssignedIdentity: ${{ secrets.AZURE_E2E_USER_ASSIGNED_IDENTITY }}
-          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
-          gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
-          gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
+          gcpClusterCreateServiceAccount: "constellation-e2e-cluster@constellation-331613.iam.gserviceaccount.com"
+          gcpIAMCreateServiceAccount: "constellation-iam-e2e@constellation-331613.iam.gserviceaccount.com"
+          gcpInClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
           test: "nop"
           buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
+          azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
 
       - name: Run upgrade test
         run: |