e2e: Enable parallel runs on Azure

.github/actions/constellation_create/action.yml

@@ -32,6 +32,9 @@ inputs:
   azureClientSecret:
     description: "The client secret value of the used secret"
     required: false
+  azureResourceGroup:
+    description: "The resource group to use for Constellation cluster"
+    required: false
 runs:
   using: "composite"
   steps:
@@ -57,7 +60,7 @@ runs:
             (.provider | select(. | has(\"azure\")).azure.tenant) = \"adb650a8-5da3-4b15-b4b0-3daf65ff7626\" |
             (.provider | select(. | has(\"azure\")).azure.location) = \"West US\" |
             (.provider | select(. | has(\"azure\")).azure.userAssignedIdentity) = \"/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/e2e-test-creds/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-test-user-assigned-id\" |
-            (.provider | select(. | has(\"azure\")).azure.resourceGroup) = \"e2e-test\" |
+            (.provider | select(. | has(\"azure\")).azure.resourceGroup) = \"${{ inputs.azureResourceGroup }}\" |
             (.provider | select(. | has(\"azure\")).azure.appClientID) = \"b657a00e-813a-4dc7-9b09-fa498a254d71\" |
             (.provider | select(. | has(\"azure\")).azure.clientSecretValue) = \"${{ inputs.azureClientSecret }}\" |
             (.provider | select(. | has(\"azure\")).azure.enforcedMeasurements) = [11,12]" \

.github/actions/e2e_cleanup/action.yml

@@ -1,10 +0,0 @@
-name: e2e test cleanup
-description: "Clean up existing resource in the e2e-test resource group on Azure"
-runs:
-  using: "composite"
-  steps:
-    - name: cleanup
-      shell: bash --noprofile --norc -e {0}
-      run: |
-        resources="$(az resource list --resource-group "e2e-test" | grep \"id\" | awk -F \" '{print $4}')"
-        for id in $resources; do az resource delete --resource-group "e2e-test" --ids "$id" --verbose; done

.github/actions/e2e_test/action.yml

@@ -30,9 +30,6 @@ inputs:
   gcpClusterServiceAccountKey:
     description: "Service account to use inside the created Constellation cluster on GCP."
     required: false
-  azure_credentials:
-    description: "Credentials authorized to create Constellation on Azure."
-    required: false
   sonobuoyTestSuiteCmd:
     description: "Which tests should be run? Check README for guidance!"
     required: true
@@ -63,6 +60,9 @@ inputs:
   azureClientSecret:
     description: "The client secret value of the used secret"
     required: false
+  azureResourceGroup:
+    description: "The resource group to use"
+    required: false
 
 runs:
   using: "composite"
@@ -83,15 +83,6 @@ runs:
       with:
         gcp_service_account_json: ${{ inputs.gcp_service_account_json }}
       if: ${{ inputs.cloudProvider == 'gcp' }}
-    - name: Login to Azure
-      uses: ./.github/actions/azure_login
-      with:
-        azure_credentials: ${{ inputs.azure_credentials }}
-      if: ${{ inputs.cloudProvider == 'azure' }}
-
-    - name: Clean resource group on Azure
-      uses: ./.github/actions/e2e_cleanup
-      if: ${{ inputs.cloudProvider == 'azure' }}
 
     - name: Create cluster
       uses: ./.github/actions/constellation_create
@@ -106,6 +97,8 @@ runs:
         isDebugImage: ${{ inputs.isDebugImage }}
         kubernetesVersion: ${{ inputs.kubernetesVersion }}
         azureClientSecret: ${{ inputs.azureClientSecret }}
+        azureResourceGroup: ${{ inputs.azureResourceGroup }}
+
     - name: Measure cluster
       uses: ./.github/actions/constellation_measure
       with:

.github/workflows/e2e-test-azure-weekly.yml

@@ -17,6 +17,21 @@ jobs:
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
         with:
           fetch-depth: 0
+
+      - name: Login to Azure
+        uses: ./.github/actions/azure_login
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
+
+      - name: Create Azure resource group
+        id: az_resource_group_gen
+        shell: bash
+        run: |
+          uuid=$(cat /proc/sys/kernel/random/uuid)
+          name=e2e-test-${uuid%%-*}
+          az group create --location westus --name $name --tags e2e
+          echo "::set-output name=res_group_name::$name"
+
       - name: Run Azure E2E test
         uses: ./.github/actions/e2e_test
         with:
@@ -25,10 +40,10 @@ jobs:
           autoscale: "false"
           cloudProvider: "azure"
           machineType: "Standard_DC4as_v5"
-          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
           sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
           kubernetesVersion: ${{ matrix.version }}
           azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
 
       - name: Always terminate cluster
         if: always()
@@ -50,3 +65,13 @@ jobs:
         shell: bash
         working-directory: .github/actions/e2e_test
 
+      - name: Always destroy Azure resource group
+        if: always()
+        shell: bash
+        run: |
+          az group delete \
+            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
+            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
+            --force-deletion-types Microsoft.Compute/virtualMachines \
+            --no-wait \
+            --yes

.github/workflows/e2e-test-azure.yml

@@ -12,6 +12,21 @@ jobs:
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
         with:
           fetch-depth: 0
+
+      - name: Login to Azure
+        uses: ./.github/actions/azure_login
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
+
+      - name: Create Azure resource group
+        id: az_resource_group_gen
+        shell: bash
+        run: |
+          uuid=$(cat /proc/sys/kernel/random/uuid)
+          name=e2e-test-${uuid%%-*}
+          az group create --location westus --name $name --tags e2e
+          echo "::set-output name=res_group_name::$name"
+
       - name: Run Azure E2E test
         uses: ./.github/actions/e2e_test
         with:
@@ -20,8 +35,8 @@ jobs:
           autoscale: "false"
           cloudProvider: "azure"
           machineType: "Standard_DC4as_v5"
-          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
           azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           # TODO: Remove E2E_SKIP once AB#2174 is resolved
           sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
           cosignPublicKey: ${{ secrets.COSIGN_PUBLIC_KEY }}
@@ -52,6 +67,17 @@ jobs:
         shell: bash
         working-directory: .github/actions/e2e_test
 
+      - name: Always destroy Azure resource group
+        if: always()
+        shell: bash
+        run: |
+          az group delete \
+            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
+            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
+            --force-deletion-types Microsoft.Compute/virtualMachines \
+            --no-wait \
+            --yes
+
   test-older-versions:
     strategy:
       matrix:
@@ -62,6 +88,21 @@ jobs:
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
         with:
           fetch-depth: 0
+
+      - name: Login to Azure
+        uses: ./.github/actions/azure_login
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
+
+      - name: Create Azure resource group
+        id: az_resource_group_gen
+        shell: bash
+        run: |
+          uuid=$(cat /proc/sys/kernel/random/uuid)
+          name=e2e-test-${uuid%%-*}
+          az group create --location westus --name $name --tags e2e
+          echo "::set-output name=res_group_name::$name"
+
       - name: Run Azure E2E test
         uses: ./.github/actions/e2e_test
         with:
@@ -72,6 +113,7 @@ jobs:
           machineType: "Standard_DC4as_v5"
           azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
           azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           sonobuoyTestSuiteCmd: "--mode quick"
           kubernetesVersion: ${{ matrix.version }}
 
@@ -95,3 +137,13 @@ jobs:
         shell: bash
         working-directory: .github/actions/e2e_test
 
+      - name: Always destroy Azure resource group
+        if: always()
+        shell: bash
+        run: |
+          az group delete \
+            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
+            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
+            --force-deletion-types Microsoft.Compute/virtualMachines \
+            --no-wait \
+            --yes

.github/workflows/e2e-test-manual.yml

@@ -56,6 +56,23 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+
+      - name: Login to Azure
+        if: ${{ github.event.inputs.cloudProvider == 'azure' }}
+        uses: ./.github/actions/azure_login
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
+
+      - name: Create Azure resource group
+        id: az_resource_group_gen
+        if: ${{ github.event.inputs.cloudProvider == 'azure' }}
+        shell: bash
+        run: |
+          uuid=$(cat /proc/sys/kernel/random/uuid)
+          name=e2e-test-${uuid%%-*}
+          az group create --location westus --name $name --tags e2e
+          echo "::set-output name=res_group_name::$name"
+
       - name: Run manual E2E test
         uses: ./.github/actions/e2e_test
         with:
@@ -66,10 +83,10 @@ jobs:
           machineType: ${{ github.event.inputs.machineType }}
           gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
           gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
-          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
           sonobuoyTestSuiteCmd: ${{ github.event.inputs.sonobuoyTestSuiteCmd }}
           kubernetesVersion: ${{ github.event.inputs.kubernetesVersion }}
           azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
           coreosImage: ${{ github.event.inputs.coreosImage }}
           isDebugImage: ${{ github.event.inputs.isDebugImage }}
           cosignPublicKey: ${{ secrets.COSIGN_PUBLIC_KEY }}
@@ -99,3 +116,13 @@ jobs:
         shell: bash
         working-directory: .github/actions/e2e_test
 
+      - name: Always destroy Azure resource group
+        if: ${{ always() && github.event.inputs.cloudProvider == 'azure' }}
+        shell: bash
+        run: |
+          az group delete \
+            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
+            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
+            --force-deletion-types Microsoft.Compute/virtualMachines \
+            --no-wait
+            --yes