From a879043f0361bc729bfba1b990fa54bf26f8f8e4 Mon Sep 17 00:00:00 2001 From: Fabian Kammel Date: Mon, 9 May 2022 09:45:59 +0200 Subject: [PATCH] E2E Test CronJob (#117) refactor e2e test into reusable action, so we can have manual & cron jobs. added cron for azure & gcp. failed jobs are reported to MS Teams. --- .../actions/constellation_destroy/action.yml | 3 + .github/actions/e2e_test/action.yml | 77 +++++++++++++++++++ .github/actions/e2e_test/teams-payload.json | 24 ++++++ .github/workflows/e2e-test-azure.yml | 25 ++++++ .github/workflows/e2e-test-gcp.yml | 25 ++++++ .../{e2e-test.yml => e2e-test-manual.yml} | 35 +++------ 6 files changed, 163 insertions(+), 26 deletions(-) create mode 100644 .github/actions/e2e_test/action.yml create mode 100644 .github/actions/e2e_test/teams-payload.json create mode 100644 .github/workflows/e2e-test-azure.yml create mode 100644 .github/workflows/e2e-test-gcp.yml rename .github/workflows/{e2e-test.yml => e2e-test-manual.yml} (74%) diff --git a/.github/actions/constellation_destroy/action.yml b/.github/actions/constellation_destroy/action.yml index a9618e100..97c131231 100644 --- a/.github/actions/constellation_destroy/action.yml +++ b/.github/actions/constellation_destroy/action.yml @@ -3,6 +3,9 @@ description: "Destroy a running Constellation cluster." runs: using: 'composite' steps: + - name: Remove VPN configuration + run: wg-quick down ./wg0.conf + shell: bash - name: Constellation terminate run: constellation terminate shell: bash diff --git a/.github/actions/e2e_test/action.yml b/.github/actions/e2e_test/action.yml new file mode 100644 index 000000000..289337425 --- /dev/null +++ b/.github/actions/e2e_test/action.yml @@ -0,0 +1,77 @@ +name: e2e_test +description: "Run Constellation e2e test." +inputs: + workerNodesCount: + description: 'Number of worker nodes to spawn.' + required: true + controlNodesCount: + description: 'Number of control-plane nodes to spawn.' + required: true + autoscale: + description: 'Autoscale?' + required: true + cloudProvider: + description: 'Which cloud provider to use.' + required: true + machineType: + description: 'VM machine type. Make sure it matches selected cloud provider!' + required: true + gcp_service_account_json: + description: 'Service account with permissions to create Constellation on GCP.' + required: false + azure_credentials: + description: 'Credentials authorized to create Constellation on Azure.' + required: false + sonobuoyTestSuiteCmd: + description: 'Which tests should be run? Check README for guidance!' + required: true + msTeamsWebhook: + description: 'WebHook used to notify of failure' + required: true +runs: + using: 'composite' + steps: + - name: Build CLI + uses: ./.github/actions/build_cli + + - name: Login to GCP + uses: ./.github/actions/gcp_login + with: + gcp_service_account_json: ${{ inputs.gcp_service_account_json }} + if: ${{ inputs.cloudProvider == 'gcp' }} + - name: Login to Azure + uses: ./.github/actions/azure_login + with: + azure_credentials: ${{ inputs.azure_credentials }} + if: ${{ inputs.cloudProvider == 'azure' }} + + - name: Create cluster + uses: ./.github/actions/constellation_create + with: + cloudProvider: ${{ inputs.cloudProvider }} + autoscale: ${{ inputs.autoscale }} + workerNodesCount: ${{ inputs.workerNodesCount }} + controlNodesCount: ${{ inputs.controlNodesCount }} + machineType: ${{ inputs.machineType }} + - name: Run e2e tests + uses: ./.github/actions/sonobuoy + with: + sonobuoyTestSuiteCmd: ${{ inputs.sonobuoyTestSuiteCmd }} + + - name: Notify teams channel + if: failure() + run: | + sudo apt-get install gettext-base -y + export TEAMS_JOB_NAME=${{ inputs.cloudProvider }} + export TEAMS_RUN_ID=${{ github.run_id }} + envsubst < teams-payload.json > to-be-send.json + curl \ + -H "Content-Type: application/json" \ + -d @to-be-send.json \ + "${{ inputs.msTeamsWebhook }}" + shell: bash + working-directory: .github/actions/e2e_test + + - name: Always terminate cluster + if: always() + uses: ./.github/actions/constellation_destroy diff --git a/.github/actions/e2e_test/teams-payload.json b/.github/actions/e2e_test/teams-payload.json new file mode 100644 index 000000000..b6c5c5c8c --- /dev/null +++ b/.github/actions/e2e_test/teams-payload.json @@ -0,0 +1,24 @@ +{ + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + "themeColor": "FF5733", + "summary": "E2E Job Failed", + "sections": [{ + "activityTitle": "E2E Job Failed", + "activitySubtitle": "${TEAMS_JOB_NAME}", + "activityImage": "https://miro.medium.com/max/552/1*G7s61tFPaLI9JRxWYpRNLw.png", + "facts": [{ + "name": "Status", + "value": "Error" + }], + "markdown": true + }], + "potentialAction": [{ + "@type": "OpenUri", + "name": "Go To Failed Action", + "targets": [{ + "os": "default", + "uri": "https://github.com/edgelesssys/constellation/actions/runs/${TEAMS_RUN_ID}" + }] + }] +} diff --git a/.github/workflows/e2e-test-azure.yml b/.github/workflows/e2e-test-azure.yml new file mode 100644 index 000000000..d74f32607 --- /dev/null +++ b/.github/workflows/e2e-test-azure.yml @@ -0,0 +1,25 @@ +name: e2e Test Azure + +on: + schedule: + - cron: '0 3 * * *' +jobs: + e2e-test-azure: + runs-on: [self-hosted, edgserver] + container: + image: ghcr.io/catthehacker/ubuntu:act-latest + options: --privileged + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Run Azure E2E test + uses: ./.github/actions/e2e_test + with: + workerNodesCount: '2' + controlNodesCount: '1' + autoscale: 'false' + cloudProvider: 'azure' + machineType: 'Standard_D4s_v3' + azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} + sonobuoyTestSuiteCmd: '--mode certified-conformance' + msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} diff --git a/.github/workflows/e2e-test-gcp.yml b/.github/workflows/e2e-test-gcp.yml new file mode 100644 index 000000000..baec26ed2 --- /dev/null +++ b/.github/workflows/e2e-test-gcp.yml @@ -0,0 +1,25 @@ +name: e2e Test GCP + +on: + schedule: + - cron: '0 4 * * *' +jobs: + e2e-test-gcp: + runs-on: [self-hosted, edgserver] + container: + image: ghcr.io/catthehacker/ubuntu:act-latest + options: --privileged + steps: + - name: Check out repository + uses: actions/checkout@v3 + - name: Run GCP E2E test + uses: ./.github/actions/e2e_test + with: + workerNodesCount: '2' + controlNodesCount: '1' + autoscale: 'false' + cloudProvider: 'gcp' + machineType: 'n2d-standard-2' + gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} + sonobuoyTestSuiteCmd: '--mode certified-conformance' + msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }} diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test-manual.yml similarity index 74% rename from .github/workflows/e2e-test.yml rename to .github/workflows/e2e-test-manual.yml index b7505f8d9..7a7158cd5 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test-manual.yml @@ -1,4 +1,4 @@ -name: e2e Test +name: e2e Test Manual on: workflow_dispatch: @@ -37,37 +37,20 @@ on: default: '--mode quick' required: true jobs: - e2e-test: + e2e-test-manual: runs-on: ubuntu-latest steps: - name: Check out repository uses: actions/checkout@v3 - - name: Build CLI - uses: ./.github/actions/build_cli - - - name: Login to GCP - uses: ./.github/actions/gcp_login + - name: Run manual E2E test + uses: ./.github/actions/e2e_test with: - gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} - if: ${{ github.event.inputs.cloudProvider == 'gcp' }} - - name: Login to Azure - uses: ./.github/actions/azure_login - with: - azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} - - - name: Create cluster - uses: ./.github/actions/constellation_create - with: - cloudProvider: ${{ github.event.inputs.cloudProvider }} - autoscale: ${{ github.event.inputs.autoscale }} workerNodesCount: ${{ github.event.inputs.workerNodesCount }} controlNodesCount: ${{ github.event.inputs.controlNodesCount }} + autoscale: ${{ github.event.inputs.autoscale }} + cloudProvider: ${{ github.event.inputs.cloudProvider }} machineType: ${{ github.event.inputs.machineType }} - - name: Run e2e tests - uses: ./.github/actions/sonobuoy - with: + gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }} + azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }} sonobuoyTestSuiteCmd: ${{ github.event.inputs.sonobuoyTestSuiteCmd }} - - - name: Always terminate cluster - if: always() - uses: ./.github/actions/constellation_destroy + msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}