From 1dad1631cace26be20003708e879f6e15d993071 Mon Sep 17 00:00:00 2001 From: Nils Hanke Date: Wed, 14 Sep 2022 17:41:47 +0200 Subject: [PATCH] E2E: Add manual macOS E2E test --- .github/actions/build_cdbg/action.yml | 14 ++ .github/actions/build_debugd/action.yml | 4 +- .../actions/constellation_create/action.yml | 1 + .github/actions/e2e_test/action.yml | 37 ++++- .../actions/generate_measurements/action.yml | 6 +- .github/workflows/build-binaries.yml | 14 ++ .github/workflows/e2e-test-manual-macos.yml | 157 ++++++++++++++++++ 7 files changed, 222 insertions(+), 11 deletions(-) create mode 100644 .github/actions/build_cdbg/action.yml create mode 100644 .github/workflows/e2e-test-manual-macos.yml diff --git a/.github/actions/build_cdbg/action.yml b/.github/actions/build_cdbg/action.yml new file mode 100644 index 000000000..29b2ddc3d --- /dev/null +++ b/.github/actions/build_cdbg/action.yml @@ -0,0 +1,14 @@ +name: Build cdbg +description: Build the Constellation cdbg binary + +runs: + using: "composite" + steps: + - name: Build cdbg + shell: bash + run: | + echo "::group::Build cdbg" + mkdir -p build && cd build + cmake .. + make cdbg + echo "::endgroup::" diff --git a/.github/actions/build_debugd/action.yml b/.github/actions/build_debugd/action.yml index 12e1b33c2..a9f473d0f 100644 --- a/.github/actions/build_debugd/action.yml +++ b/.github/actions/build_debugd/action.yml @@ -7,7 +7,7 @@ inputs: default: "./debugd" required: true -# Linux runner only +# Linux runner only (homedir trick does not work on macOS, required for private runner) runs: using: "composite" steps: @@ -22,6 +22,6 @@ runs: export GOPATH=${homedir}/go export GOPRIVATE=github.com/edgelesssys export GOMODCACHE=${homedir}/.cache/go-mod - make debugd cdbg + make debugd mv -n debugd "${{ inputs.outputPath }}" echo "::endgroup::" diff --git a/.github/actions/constellation_create/action.yml b/.github/actions/constellation_create/action.yml index 480718fd9..a1cb05bc0 100644 --- a/.github/actions/constellation_create/action.yml +++ b/.github/actions/constellation_create/action.yml @@ -141,6 +141,7 @@ runs: - name: Cdbg deploy run: | echo "::group::cdbg deploy" + chmod +x $GITHUB_WORKSPACE/build/cdbg cdbg deploy --bootstrapper $GITHUB_WORKSPACE/build/bootstrapper echo "::endgroup::" shell: bash diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml index 6505259f9..4298eaa1f 100644 --- a/.github/actions/e2e_test/action.yml +++ b/.github/actions/e2e_test/action.yml @@ -46,22 +46,47 @@ inputs: runs: using: "composite" steps: + - name: Determine build target + id: determine-build-target + shell: bash + run: | + echo "::set-output name=hostOS::$(go env GOOS)" + echo "::set-output name=hostArch::$(go env GOARCH)" + - name: Build CLI uses: ./.github/actions/build_cli + with: + targetOS: ${{ steps.determine-build-target.outputs.hostOS }} + targetArch: ${{ steps.determine-build-target.outputs.hostArch }} + + # macOS runners don't have Docker preinstalled, so they cannot build the bootstrapper. + # But we can use a Linux runner to build it and store/retrieve it from the action cache. + - name: Download the bootstrapper from cache + id: download-bootstrapper-cache + if: inputs.isDebugImage == 'true' && runner.os == 'macOS' + uses: actions/cache@v3 + with: + key: bootstrapper-${{ github.sha }} + path: "build/bootstrapper" + - name: Build the bootstrapper id: build-bootstrapper uses: ./.github/actions/build_bootstrapper - if: ${{ inputs.isDebugImage == 'true' }} - - name: Build debugd - id: build-debugd - uses: ./.github/actions/build_debugd - if: ${{ inputs.isDebugImage == 'true' }} + if: inputs.isDebugImage == 'true' && runner.os != 'macOS' + + - name: Build cdbg + id: build-cdbg + uses: ./.github/actions/build_cdbg + if: inputs.isDebugImage == 'true' + with: + targetOS: ${{ steps.determine-build-target.outputs.hostOS }} + targetArch: ${{ steps.determine-build-target.outputs.hostArch }} - name: Login to GCP uses: ./.github/actions/gcp_login with: gcp_service_account_json: ${{ inputs.gcp_service_account_json }} - if: ${{ inputs.cloudProvider == 'gcp' }} + if: inputs.cloudProvider == 'gcp' - name: Create cluster uses: ./.github/actions/constellation_create diff --git a/.github/actions/generate_measurements/action.yml b/.github/actions/generate_measurements/action.yml index ef863ca13..1df6e8597 100644 --- a/.github/actions/generate_measurements/action.yml +++ b/.github/actions/generate_measurements/action.yml @@ -72,9 +72,9 @@ runs: id: build-bootstrapper uses: ./.github/actions/build_bootstrapper if: ${{ inputs.isDebugImage == 'true' }} - - name: Build debugd - id: build-debugd - uses: ./.github/actions/build_debugd + - name: Build cdbg + id: build-cdbg + uses: ./.github/actions/build_cdbg if: ${{ inputs.isDebugImage == 'true' }} - name: Login to GCP diff --git a/.github/workflows/build-binaries.yml b/.github/workflows/build-binaries.yml index fbc93b346..c6d9770fd 100644 --- a/.github/workflows/build-binaries.yml +++ b/.github/workflows/build-binaries.yml @@ -46,6 +46,20 @@ jobs: - name: Build debugd uses: ./.github/actions/build_debugd + build-cdbg: + runs-on: ubuntu-latest + steps: + - name: Setup Go environment + uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f + with: + go-version: "1.19.1" + + - name: Check out repository + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Build cdbg + uses: ./.github/actions/build_cdbg + build-disk-mapper: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/e2e-test-manual-macos.yml b/.github/workflows/e2e-test-manual-macos.yml new file mode 100644 index 000000000..50bf8e168 --- /dev/null +++ b/.github/workflows/e2e-test-manual-macos.yml @@ -0,0 +1,157 @@ +name: e2e Test Manual (macOS CLI) + +on: + workflow_dispatch: + inputs: + workerNodesCount: + description: "Number of worker nodes to spawn." + default: "2" + required: true + controlNodesCount: + description: "Number of control-plane nodes to spawn." + default: "1" + required: true + autoscale: + description: "Autoscale?" + type: boolean + default: false + required: true + cloudProvider: + description: "Which cloud provider to use." + type: choice + options: + - "gcp" + - "azure" + default: "gcp" + required: true + sonobuoyTestSuiteCmd: + description: "Which tests should be run? Check README for guidance!" + default: "--mode quick" + required: true + kubernetesVersion: + description: "Kubernetes version to create the cluster from." + default: "1.24" + required: true + coreosImage: + description: "CoreOS image (full path). Examples are in internal/config/config.go." + default: "debug-latest" + type: string + required: true + isDebugImage: + description: "Is CoreOS image a debug image?" + type: boolean + default: true + required: false + machineType: + description: "Override VM machine type. Leave as 'default' or empty to use the default VM type for the selected cloud provider." + type: string + default: "default" + required: false + +jobs: + build-bootstrapper-linux: + name: "Build bootstrapper (debug image)" + runs-on: ubuntu-latest + if: ${{ github.event.inputs.isDebugImage == 'true' }} + steps: + - name: Setup Go environment + uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f + if: ${{ github.event.steps.check-bootstrapper-cache.cache-hit != 'true'}} + with: + go-version: "1.19.1" + + - name: Check out repository + if: ${{ github.event.steps.check-bootstrapper-cache.cache-hit != 'true'}} + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Build bootstrapper + if: ${{ github.event.steps.check-bootstrapper-cache.cache-hit != 'true'}} + uses: ./.github/actions/build_bootstrapper + + - name: Upload bootstrapper to cache + if: ${{ github.event.steps.check-bootstrapper-cache.cache-hit != 'true'}} + uses: actions/cache@v3 + with: + key: bootstrapper-${{ github.sha }} + path: "build/bootstrapper" + + e2e-test-manual-darwin: + name: "e2e Test Manual (macOS)" + runs-on: macos-12 + needs: build-bootstrapper-linux + if: ${{ always() && !cancelled() && (needs.build-bootstrapper-linux.result == 'success' || needs.build-bootstrapper-linux.result == 'skipped') }} + steps: + - name: Check out repository + uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b + + - name: Setup Go environment + uses: actions/setup-go@268d8c0ca0432bb2cf416faae41297df9d262d7f + with: + go-version: "1.19.1" + + - name: Login to Azure + if: ${{ github.event.inputs.cloudProvider == 'azure' }} + uses: ./.github/actions/azure_login + with: + azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} + + - name: Create Azure resource group + id: az_resource_group_gen + if: ${{ github.event.inputs.cloudProvider == 'azure' }} + shell: bash + run: | + uuid=$(uuidgen) + name=e2e-test-${uuid%%-*} + az group create --location westus --name $name --tags e2e + echo "::set-output name=res_group_name::$name" + + - name: Set up gcloud CLI + if: ${{ github.event.inputs.cloudProvider == 'gcp' }} + uses: google-github-actions/setup-gcloud@877d4953d2c70a0ba7ef3290ae968eb24af233bb + + - name: Run manual E2E test + uses: ./.github/actions/e2e_test + with: + workerNodesCount: ${{ github.event.inputs.workerNodesCount }} + controlNodesCount: ${{ github.event.inputs.controlNodesCount }} + autoscale: ${{ github.event.inputs.autoscale }} + cloudProvider: ${{ github.event.inputs.cloudProvider }} + machineType: ${{ github.event.inputs.machineType }} + gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} + gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }} + sonobuoyTestSuiteCmd: ${{ github.event.inputs.sonobuoyTestSuiteCmd }} + kubernetesVersion: ${{ github.event.inputs.kubernetesVersion }} + azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }} + azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }} + coreosImage: ${{ github.event.inputs.coreosImage }} + isDebugImage: ${{ github.event.inputs.isDebugImage }} + + - name: Always terminate cluster + if: always() + continue-on-error: true + uses: ./.github/actions/constellation_destroy + + - name: Notify teams channel + if: ${{ failure() && github.ref == 'refs/heads/main' }} + run: | + brew install gettext && brew link --force gettext + export TEAMS_JOB_NAME="${{ github.event.inputs.cloudProvider }} (macOS, manual)" + export TEAMS_RUN_ID=${{ github.run_id }} + envsubst < teams-payload.json > to-be-send.json + curl \ + -H "Content-Type: application/json" \ + -d @to-be-send.json \ + "${{ secrets.MS_TEAMS_WEBHOOK_URI }}" + shell: bash + working-directory: .github/actions/e2e_test + + - name: Always destroy Azure resource group + if: ${{ always() && github.event.inputs.cloudProvider == 'azure' }} + shell: bash + run: | + az group delete \ + --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \ + --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \ + --force-deletion-types Microsoft.Compute/virtualMachines \ + --no-wait \ + --yes