From ef8130a918c2aab803b75421260d923800f3e8cf Mon Sep 17 00:00:00 2001 From: katexochen <49727155+katexochen@users.noreply.github.com> Date: Fri, 2 Sep 2022 12:43:04 +0200 Subject: [PATCH] e2e: Enable parallel runs on Azure --- .../actions/constellation_create/action.yml | 5 +- .github/actions/e2e_cleanup/action.yml | 10 ---- .github/actions/e2e_test/action.yml | 17 ++---- .github/workflows/e2e-test-azure-weekly.yml | 27 +++++++++- .github/workflows/e2e-test-azure.yml | 54 ++++++++++++++++++- .github/workflows/e2e-test-manual.yml | 29 +++++++++- 6 files changed, 116 insertions(+), 26 deletions(-) delete mode 100644 .github/actions/e2e_cleanup/action.yml diff --git a/.github/actions/constellation_create/action.yml b/.github/actions/constellation_create/action.yml index 09917ee70..7618c3fb3 100644 --- a/.github/actions/constellation_create/action.yml +++ b/.github/actions/constellation_create/action.yml @@ -32,6 +32,9 @@ inputs: azureClientSecret: description: "The client secret value of the used secret" required: false + azureResourceGroup: + description: "The resource group to use for Constellation cluster" + required: false runs: using: "composite" steps: @@ -57,7 +60,7 @@ runs: (.provider | select(. | has(\"azure\")).azure.tenant) = \"adb650a8-5da3-4b15-b4b0-3daf65ff7626\" | (.provider | select(. | has(\"azure\")).azure.location) = \"West US\" | (.provider | select(. | has(\"azure\")).azure.userAssignedIdentity) = \"/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/e2e-test-creds/providers/Microsoft.ManagedIdentity/userAssignedIdentities/e2e-test-user-assigned-id\" | - (.provider | select(. | has(\"azure\")).azure.resourceGroup) = \"e2e-test\" | + (.provider | select(. | has(\"azure\")).azure.resourceGroup) = \"${{ inputs.azureResourceGroup }}\" | (.provider | select(. | has(\"azure\")).azure.appClientID) = \"b657a00e-813a-4dc7-9b09-fa498a254d71\" | (.provider | select(. | has(\"azure\")).azure.clientSecretValue) = \"${{ inputs.azureClientSecret }}\" | (.provider | select(. | has(\"azure\")).azure.enforcedMeasurements) = [11,12]" \ diff --git a/.github/actions/e2e_cleanup/action.yml b/.github/actions/e2e_cleanup/action.yml deleted file mode 100644 index cab23f6b9..000000000 --- a/.github/actions/e2e_cleanup/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: e2e test cleanup -description: "Clean up existing resource in the e2e-test resource group on Azure" -runs: - using: "composite" - steps: - - name: cleanup - shell: bash --noprofile --norc -e {0} - run: | - resources="$(az resource list --resource-group "e2e-test" | grep \"id\" | awk -F \" '{print $4}')" - for id in $resources; do az resource delete --resource-group "e2e-test" --ids "$id" --verbose; done diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml index 48f77a94c..232873e70 100644 --- a/.github/actions/e2e_test/action.yml +++ b/.github/actions/e2e_test/action.yml @@ -30,9 +30,6 @@ inputs: gcpClusterServiceAccountKey: description: "Service account to use inside the created Constellation cluster on GCP." required: false - azure_credentials: - description: "Credentials authorized to create Constellation on Azure." - required: false sonobuoyTestSuiteCmd: description: "Which tests should be run? Check README for guidance!" required: true @@ -63,6 +60,9 @@ inputs: azureClientSecret: description: "The client secret value of the used secret" required: false + azureResourceGroup: + description: "The resource group to use" + required: false runs: using: "composite" @@ -83,15 +83,6 @@ runs: with: gcp_service_account_json: ${{ inputs.gcp_service_account_json }} if: ${{ inputs.cloudProvider == 'gcp' }} - - name: Login to Azure - uses: ./.github/actions/azure_login - with: - azure_credentials: ${{ inputs.azure_credentials }} - if: ${{ inputs.cloudProvider == 'azure' }} - - - name: Clean resource group on Azure - uses: ./.github/actions/e2e_cleanup - if: ${{ inputs.cloudProvider == 'azure' }} - name: Create cluster uses: ./.github/actions/constellation_create @@ -106,6 +97,8 @@ runs: isDebugImage: ${{ inputs.isDebugImage }} kubernetesVersion: ${{ inputs.kubernetesVersion }} azureClientSecret: ${{ inputs.azureClientSecret }} + azureResourceGroup: ${{ inputs.azureResourceGroup }} + - name: Measure cluster uses: ./.github/actions/constellation_measure with: diff --git a/.github/workflows/e2e-test-azure-weekly.yml b/.github/workflows/e2e-test-azure-weekly.yml index 8dae05e43..8c307911d 100644 --- a/.github/workflows/e2e-test-azure-weekly.yml +++ b/.github/workflows/e2e-test-azure-weekly.yml @@ -17,6 +17,21 @@ jobs: uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b with: fetch-depth: 0 + + - name: Login to Azure + uses: ./.github/actions/azure_login + with: + azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} + + - name: Create Azure resource group + id: az_resource_group_gen + shell: bash + run: | + uuid=$(cat /proc/sys/kernel/random/uuid) + name=e2e-test-${uuid%%-*} + az group create --location westus --name $name --tags e2e + echo "::set-output name=res_group_name::$name" + - name: Run Azure E2E test uses: ./.github/actions/e2e_test with: @@ -25,10 +40,10 @@ jobs: autoscale: "false" cloudProvider: "azure" machineType: "Standard_DC4as_v5" - azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml' kubernetesVersion: ${{ matrix.version }} azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }} + azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }} - name: Always terminate cluster if: always() @@ -50,3 +65,13 @@ jobs: shell: bash working-directory: .github/actions/e2e_test + - name: Always destroy Azure resource group + if: always() + shell: bash + run: | + az group delete \ + --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \ + --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \ + --force-deletion-types Microsoft.Compute/virtualMachines \ + --no-wait \ + --yes diff --git a/.github/workflows/e2e-test-azure.yml b/.github/workflows/e2e-test-azure.yml index 56fed52f4..69ad8af5d 100644 --- a/.github/workflows/e2e-test-azure.yml +++ b/.github/workflows/e2e-test-azure.yml @@ -12,6 +12,21 @@ jobs: uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b with: fetch-depth: 0 + + - name: Login to Azure + uses: ./.github/actions/azure_login + with: + azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} + + - name: Create Azure resource group + id: az_resource_group_gen + shell: bash + run: | + uuid=$(cat /proc/sys/kernel/random/uuid) + name=e2e-test-${uuid%%-*} + az group create --location westus --name $name --tags e2e + echo "::set-output name=res_group_name::$name" + - name: Run Azure E2E test uses: ./.github/actions/e2e_test with: @@ -20,8 +35,8 @@ jobs: autoscale: "false" cloudProvider: "azure" machineType: "Standard_DC4as_v5" - azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }} + azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }} # TODO: Remove E2E_SKIP once AB#2174 is resolved sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml' cosignPublicKey: ${{ secrets.COSIGN_PUBLIC_KEY }} @@ -52,6 +67,17 @@ jobs: shell: bash working-directory: .github/actions/e2e_test + - name: Always destroy Azure resource group + if: always() + shell: bash + run: | + az group delete \ + --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \ + --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \ + --force-deletion-types Microsoft.Compute/virtualMachines \ + --no-wait \ + --yes + test-older-versions: strategy: matrix: @@ -62,6 +88,21 @@ jobs: uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b with: fetch-depth: 0 + + - name: Login to Azure + uses: ./.github/actions/azure_login + with: + azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} + + - name: Create Azure resource group + id: az_resource_group_gen + shell: bash + run: | + uuid=$(cat /proc/sys/kernel/random/uuid) + name=e2e-test-${uuid%%-*} + az group create --location westus --name $name --tags e2e + echo "::set-output name=res_group_name::$name" + - name: Run Azure E2E test uses: ./.github/actions/e2e_test with: @@ -72,6 +113,7 @@ jobs: machineType: "Standard_DC4as_v5" azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }} + azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }} sonobuoyTestSuiteCmd: "--mode quick" kubernetesVersion: ${{ matrix.version }} @@ -95,3 +137,13 @@ jobs: shell: bash working-directory: .github/actions/e2e_test + - name: Always destroy Azure resource group + if: always() + shell: bash + run: | + az group delete \ + --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \ + --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \ + --force-deletion-types Microsoft.Compute/virtualMachines \ + --no-wait \ + --yes diff --git a/.github/workflows/e2e-test-manual.yml b/.github/workflows/e2e-test-manual.yml index d0d403eac..6b9f1c4e5 100644 --- a/.github/workflows/e2e-test-manual.yml +++ b/.github/workflows/e2e-test-manual.yml @@ -56,6 +56,23 @@ jobs: steps: - name: Check out repository uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Login to Azure + if: ${{ github.event.inputs.cloudProvider == 'azure' }} + uses: ./.github/actions/azure_login + with: + azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} + + - name: Create Azure resource group + id: az_resource_group_gen + if: ${{ github.event.inputs.cloudProvider == 'azure' }} + shell: bash + run: | + uuid=$(cat /proc/sys/kernel/random/uuid) + name=e2e-test-${uuid%%-*} + az group create --location westus --name $name --tags e2e + echo "::set-output name=res_group_name::$name" + - name: Run manual E2E test uses: ./.github/actions/e2e_test with: @@ -66,10 +83,10 @@ jobs: machineType: ${{ github.event.inputs.machineType }} gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }} - azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} sonobuoyTestSuiteCmd: ${{ github.event.inputs.sonobuoyTestSuiteCmd }} kubernetesVersion: ${{ github.event.inputs.kubernetesVersion }} azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }} + azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }} coreosImage: ${{ github.event.inputs.coreosImage }} isDebugImage: ${{ github.event.inputs.isDebugImage }} cosignPublicKey: ${{ secrets.COSIGN_PUBLIC_KEY }} @@ -99,3 +116,13 @@ jobs: shell: bash working-directory: .github/actions/e2e_test + - name: Always destroy Azure resource group + if: ${{ always() && github.event.inputs.cloudProvider == 'azure' }} + shell: bash + run: | + az group delete \ + --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \ + --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \ + --force-deletion-types Microsoft.Compute/virtualMachines \ + --no-wait + --yes