From ec6b20a4c6c4b1ac3b7ebf041bdd78f5105faa68 Mon Sep 17 00:00:00 2001 From: Moritz Sanft <58110325+msanft@users.noreply.github.com> Date: Fri, 15 Dec 2023 09:03:22 +0100 Subject: [PATCH] ci: parallelize upgrade e2e test Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com> --- .github/actions/artifact_download/action.yml | 4 +- .github/actions/artifact_upload/action.yml | 4 +- .../actions/constellation_create/action.yml | 4 +- .github/actions/e2e_benchmark/action.yml | 8 +- .github/actions/e2e_sonobuoy/action.yml | 4 +- .github/actions/e2e_test/action.yml | 10 +- .github/workflows/e2e-test-daily.yml | 2 +- .github/workflows/e2e-test-release.yml | 2 +- .github/workflows/e2e-test-weekly.yml | 2 +- .github/workflows/e2e-test.yml | 2 +- .github/workflows/e2e-upgrade.yml | 255 +++++++++++++++--- 11 files changed, 244 insertions(+), 53 deletions(-) diff --git a/.github/actions/artifact_download/action.yml b/.github/actions/artifact_download/action.yml index 5db482d3d..9eb16d77d 100644 --- a/.github/actions/artifact_download/action.yml +++ b/.github/actions/artifact_download/action.yml @@ -9,7 +9,7 @@ inputs: description: 'Download to a specified path.' required: false default: ./ - encryptionSecret: + encryption-secret: description: 'The secret to use for decrypting the artifact.' required: true @@ -37,4 +37,4 @@ runs: shell: bash run: | mkdir -p ${{ inputs.path }} - unzip -P '${{ inputs.encryptionSecret }}' -qq -d ${{ inputs.path }} ${{ steps.tempdir.outputs.directory }}/archive.zip + unzip -P '${{ inputs.encryption-secret }}' -qq -d ${{ inputs.path }} ${{ steps.tempdir.outputs.directory }}/archive.zip diff --git a/.github/actions/artifact_upload/action.yml b/.github/actions/artifact_upload/action.yml index 8de9b113a..f6a5ec9bf 100644 --- a/.github/actions/artifact_upload/action.yml +++ b/.github/actions/artifact_upload/action.yml @@ -11,7 +11,7 @@ inputs: retention-days: description: 'How long the artifact should be retained for.' default: 60 - encryptionSecret: + encryption-secret: description: 'The secret to use for encrypting the files.' required: true @@ -58,7 +58,7 @@ runs: for target in ${paths} do pushd "$(dirname "${target}")" || exit 1 - zip -e -P '${{ inputs.encryptionSecret }}' -r "${{ steps.tempdir.outputs.directory }}/archive.zip" "$(basename "${target}")" + zip -e -P '${{ inputs.encryption-secret }}' -r "${{ steps.tempdir.outputs.directory }}/archive.zip" "$(basename "${target}")" popd || exit 1 done diff --git a/.github/actions/constellation_create/action.yml b/.github/actions/constellation_create/action.yml index fa2d8f7e3..0c9c6e6d2 100644 --- a/.github/actions/constellation_create/action.yml +++ b/.github/actions/constellation_create/action.yml @@ -59,7 +59,7 @@ inputs: force: description: "Set the force-flag on apply to ignore version mismatches." required: false - encryptionSecret: + encryption-secret: description: "The secret to use for encrypting the artifact." required: true @@ -274,4 +274,4 @@ runs: name: serial-logs-${{ inputs.artifactNameSuffix }} path: > !(terraform).log - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} diff --git a/.github/actions/e2e_benchmark/action.yml b/.github/actions/e2e_benchmark/action.yml index afc9aed38..b08285629 100644 --- a/.github/actions/e2e_benchmark/action.yml +++ b/.github/actions/e2e_benchmark/action.yml @@ -17,7 +17,7 @@ inputs: awsOpenSearchPwd: description: "AWS OpenSearch Password to upload the results." required: false - encryptionSecret: + encryption-secret: description: 'The secret to use for encrypting the artifact.' required: true @@ -100,7 +100,7 @@ runs: with: path: "out/fio-constellation-${{ inputs.cloudProvider }}.json" name: "fio-constellation-${{ inputs.cloudProvider }}.json" - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} - name: Run knb benchmark shell: bash @@ -122,7 +122,7 @@ runs: with: path: "out/knb-constellation-${{ inputs.cloudProvider }}.json" name: "knb-constellation-${{ inputs.cloudProvider }}.json" - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} - name: Assume AWS role to retrieve and update benchmarks in S3 uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 @@ -176,7 +176,7 @@ runs: path: > benchmarks/constellation-${{ inputs.cloudProvider }}.json name: "benchmarks" - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} - name: Upload benchmark results to opensearch if: (!env.ACT) diff --git a/.github/actions/e2e_sonobuoy/action.yml b/.github/actions/e2e_sonobuoy/action.yml index 7b87d0ee1..7c129a692 100644 --- a/.github/actions/e2e_sonobuoy/action.yml +++ b/.github/actions/e2e_sonobuoy/action.yml @@ -11,7 +11,7 @@ inputs: kubeconfig: description: "The kubeconfig of the cluster to test." required: true - encryptionSecret: + encryption-secret: description: 'The secret to use for encrypting the artifact.' required: true @@ -54,7 +54,7 @@ runs: with: name: "sonobuoy-logs-${{ inputs.artifactNameSuffix }}.tar.gz" path: "*_sonobuoy_*.tar.gz" - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} # Only works on "sonobuoy full" tests (e2e plugin) - name: Extract test results diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml index 99452fa46..c1532200f 100644 --- a/.github/actions/e2e_test/action.yml +++ b/.github/actions/e2e_test/action.yml @@ -86,7 +86,7 @@ inputs: force: description: "Set the force-flag on apply to ignore version mismatches." required: false - encryptionSecret: + encryption-secret: description: 'The secret to use for decrypting the artifact.' required: true @@ -299,7 +299,7 @@ runs: clusterCreation: ${{ inputs.clusterCreation }} marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }} force: ${{ inputs.force }} - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} - name: Deploy log- and metrics-collection (Kubernetes) id: deploy-logcollection @@ -334,7 +334,7 @@ runs: sonobuoyTestSuiteCmd: "--mode quick" kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }} - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} - name: Run sonobuoy full test if: inputs.test == 'sonobuoy full' @@ -344,7 +344,7 @@ runs: sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol|Services should serve endpoints on same port and different protocols" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml' kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }} artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }} - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} - name: Run autoscaling test if: inputs.test == 'autoscaling' @@ -367,7 +367,7 @@ runs: awsOpenSearchDomain: ${{ inputs.awsOpenSearchDomain }} awsOpenSearchUsers: ${{ inputs.awsOpenSearchUsers }} awsOpenSearchPwd: ${{ inputs.awsOpenSearchPwd }} - encryptionSecret: ${{ inputs.encryptionSecret }} + encryption-secret: ${{ inputs.encryption-secret }} - name: Run constellation verify test if: inputs.test == 'verify' diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index 2a94d260f..3514a887f 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -91,7 +91,7 @@ jobs: awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }} awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }} clusterCreation: "cli" - encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Always terminate cluster if: always() diff --git a/.github/workflows/e2e-test-release.yml b/.github/workflows/e2e-test-release.yml index e1229b088..06a89138b 100644 --- a/.github/workflows/e2e-test-release.yml +++ b/.github/workflows/e2e-test-release.yml @@ -265,7 +265,7 @@ jobs: clusterCreation: ${{ matrix.clusterCreation }} s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }} s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }} - encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Always terminate cluster if: always() diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index 4de0bb617..759e7e3e3 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -305,7 +305,7 @@ jobs: clusterCreation: ${{ matrix.clusterCreation }} s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }} s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }} - encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Always terminate cluster if: always() diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 2218a3270..902dd24da 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -246,7 +246,7 @@ jobs: s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }} marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }} force: ${{ inputs.force }} - encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Always terminate cluster if: always() diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml index 1a0311f7d..9002f8066 100644 --- a/.github/workflows/e2e-upgrade.yml +++ b/.github/workflows/e2e-upgrade.yml @@ -111,7 +111,57 @@ jobs: echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT" echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT" - e2e-upgrade: + build-target-cli: + name: Build upgrade target version CLI + runs-on: ubuntu-22.04 + permissions: + id-token: write + checks: write + contents: read + packages: write + steps: + - name: Checkout + if: inputs.gitRef == 'head' + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + fetch-depth: 0 + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Checkout ref + if: inputs.gitRef != 'head' + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + fetch-depth: 0 + ref: ${{ inputs.gitRef }} + + - name: Setup Bazel & Nix + uses: ./.github/actions/setup_bazel_nix + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + + - name: Log in to the Container registry + uses: ./.github/actions/container_registry_login + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build CLI + uses: ./.github/actions/build_cli + with: + enterpriseCLI: true + outputPath: "build/constellation" + push: true + + - name: Upload CLI binary + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: constellation + path: build/constellation + + create-cluster: + name: Create upgrade origin version cluster runs-on: ubuntu-22.04 permissions: id-token: write @@ -119,6 +169,8 @@ jobs: contents: read packages: write needs: [split-nodeCount] + outputs: + kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} steps: - name: Checkout if: inputs.gitRef == 'head' @@ -139,26 +191,6 @@ jobs: useCache: "true" buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} - - name: Login to AWS - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead - aws-region: eu-central-1 - - - name: Find latest nightly image - id: find-image - if: inputs.toImage == '' - uses: ./.github/actions/versionsapi - with: - command: latest - ref: main - stream: nightly - - - name: Simulate patch upgrade - if: inputs.simulatedTargetVersion != '' - run: | - echo ${{ inputs.simulatedTargetVersion }} > version.txt - - name: Create cluster with 'fromVersion' CLI. id: e2e_test uses: ./.github/actions/e2e_test @@ -183,14 +215,80 @@ jobs: awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }} awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }} clusterCreation: "cli" - encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - - name: Build CLI - uses: ./.github/actions/build_cli + - name: Remove Terraform plugin cache + if: always() + run: | + rm -rf constellation-terraform/.terraform + rm -rf constellation-iam-terraform/.terraform + + - name: Upload Working Directory + if: always() + uses: ./.github/actions/artifact_upload with: - enterpriseCLI: true - outputPath: "build/constellation" - push: true + name: constellation-pre-test + path: > + ${{ steps.e2e_test.outputs.kubeconfig }} + constellation-terraform + constellation-iam-terraform + constellation-conf.yaml + constellation-state.yaml + constellation-mastersecret.json + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + e2e-upgrade: + name: Run upgrade test + runs-on: ubuntu-22.04 + permissions: + id-token: write + checks: write + contents: read + packages: write + needs: + - split-nodeCount + - build-target-cli + - create-cluster + steps: + - name: Checkout + if: inputs.gitRef == 'head' + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + fetch-depth: 0 + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Checkout ref + if: inputs.gitRef != 'head' + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + fetch-depth: 0 + ref: ${{ inputs.gitRef }} + + - name: Setup Bazel & Nix + uses: ./.github/actions/setup_bazel_nix + with: + useCache: "true" + buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }} + + - name: Login to AWS + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + with: + role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead + aws-region: eu-central-1 + + - name: Find latest nightly image + id: find-image + if: inputs.toImage == '' + uses: ./.github/actions/versionsapi + with: + command: latest + ref: main + stream: nightly + + - name: Simulate patch upgrade + if: inputs.simulatedTargetVersion != '' + run: | + echo ${{ inputs.simulatedTargetVersion }} > version.txt - name: Login to GCP (IAM service account) if: inputs.cloudProvider == 'gcp' @@ -213,6 +311,25 @@ jobs: with: azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} + - name: Download CLI + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: constellation + path: build + + - name: Download Working Directory (Pre-test) + uses: ./.github/actions/artifact_download + with: + name: constellation-pre-test + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Make Constellation executable and add to PATH + if: always() + run: | + chmod +x build/constellation + export PATH="$PATH:build" + echo "build" >> "$GITHUB_PATH" + - name: Migrate config id: constellation-config-migrate run: | @@ -245,7 +362,7 @@ jobs: - name: Run upgrade test env: - KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }} + KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }} IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }} KUBERNETES: ${{ inputs.toKubernetes }} MICROSERVICES: ${{ inputs.toMicroservices }} @@ -265,10 +382,82 @@ jobs: bazel run //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" + - name: Remove Terraform plugin cache + if: always() + run: | + rm -rf constellation-terraform/.terraform + rm -rf constellation-iam-terraform/.terraform + + - name: Upload Working Directory + if: always() + uses: ./.github/actions/artifact_upload + with: + name: constellation-post-test + path: | + ${{ needs.create-cluster.outputs.kubeconfig }} + constellation-terraform + constellation-iam-terraform + constellation-conf.yaml + constellation-state.yaml + constellation-mastersecret.json + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + clean-up: + name: Clean up resources + runs-on: ubuntu-22.04 + permissions: + id-token: write + checks: write + contents: read + packages: write + if: always() + needs: [create-cluster, e2e-upgrade] + steps: + - name: Checkout + if: inputs.gitRef == 'head' + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + fetch-depth: 0 + ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} + + - name: Checkout ref + if: inputs.gitRef != 'head' + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + fetch-depth: 0 + ref: ${{ inputs.gitRef }} + + - name: Download CLI + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: constellation + path: build + + - name: Download Working Directory (Pre-test) + if: always() && needs.e2e-upgrade.result != 'success' + uses: ./.github/actions/artifact_download + with: + name: constellation-pre-test + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Download Working Directory (Post-test) + if: always() && needs.e2e-upgrade.result == 'success' + uses: ./.github/actions/artifact_download + with: + name: constellation-post-test + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + + - name: Make Constellation executable and add to PATH + if: always() + run: | + chmod +x build/constellation + export PATH="$PATH:build" + echo "build" >> "$GITHUB_PATH" + - name: Always fetch logs if: always() env: - KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }} + KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }} run: | kubectl logs -n kube-system -l "app.kubernetes.io/name=constellation-operator" --tail=-1 > node-operator.logs kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs @@ -283,14 +472,14 @@ jobs: node-operator.logs node-maintenance-operator.logs constellation-version.yaml - encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} + encryption-secret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Always terminate cluster if: always() uses: ./.github/actions/constellation_destroy with: - kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} clusterCreation: "cli" + kubeconfig: ${{ needs.create-cluster.outputs.kubeconfig }} cloudProvider: ${{ inputs.cloudProvider }} azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" @@ -305,7 +494,9 @@ jobs: - name: Notify about failure if: | - failure() && + always() && + needs.create-cluster.result != 'success' && + needs.e2e-upgrade.result != 'success' && github.ref == 'refs/heads/main' && inputs.scheduled continue-on-error: true