AB#2266: Test all supported version with e2e-tests

* e2e-test workflows execute two hours earlier.
* Run quick-mode e2e tests for the two older versions we support.
This triggers every night, together with the existing e2e tests.
Idea here is that we know that a cluster can be setup and initialized.
* Run full e2e tests for the two older versions each sunday.
* Do not abort manual e2e runs. This allows for parallel runs.
* Run unprivileged container

.github/actions/constellation_create/action.yml

@@ -17,8 +17,11 @@ inputs:
   machineType:
     description: "Machine type of VM to spawn."
     required: true
+  kubernetesVersion:
+    description: "Kubernetes version to create the cluster from."
+    required: false
 runs:
-  using: 'composite'
+  using: "composite"
   steps:
     - name: Install kubectl
       run: |
@@ -36,6 +39,7 @@ runs:
     - name: Constellation config generate
       run: |
         constellation config generate ${{ inputs.cloudProvider }}
+
         yq eval -i \
           "(.provider | select(. | has(\"azure\")).azure.subscription) = \"0d202bbb-4fa7-4af8-8125-58c269a05435\" |
             (.provider | select(. | has(\"azure\")).azure.tenant) = \"adb650a8-5da3-4b15-b4b0-3daf65ff7626\" |
@@ -47,6 +51,10 @@ runs:
             (.provider | select(. | has(\"gcp\")).gcp.region) = \"europe-west3\" |
             (.provider | select(. | has(\"gcp\")).gcp.zone) = \"europe-west3-b\"" \
           constellation-conf.yaml
+
+        if [ ${{ inputs.kubernetesVersion != '' }} = true ]; then
+          yq eval -i "(.kubernetesVersion) = ${{ inputs.kubernetesVersion }}" constellation-conf.yaml
+        fi
       shell: bash
 
     - name: Set latest image

.github/actions/e2e_test/action.yml

@@ -2,55 +2,58 @@ name: E2E test
 description: "Run Constellation e2e test."
 inputs:
   workerNodesCount:
-    description: 'Number of worker nodes to spawn.'
+    description: "Number of worker nodes to spawn."
     required: true
   controlNodesCount:
-    description: 'Number of control-plane nodes to spawn.'
+    description: "Number of control-plane nodes to spawn."
     required: true
   autoscale:
-    description: 'Autoscale?'
+    description: "Autoscale?"
     required: true
   cloudProvider:
-    description: 'Which cloud provider to use.'
+    description: "Which cloud provider to use."
     required: true
   machineType:
-    description: 'VM machine type. Make sure it matches selected cloud provider!'
+    description: "VM machine type. Make sure it matches selected cloud provider!"
     required: true
   gcp_service_account_json:
-    description: 'Service account with permissions to create Constellation on GCP.'
+    description: "Service account with permissions to create Constellation on GCP."
     required: false
   azure_credentials:
-    description: 'Credentials authorized to create Constellation on Azure.'
+    description: "Credentials authorized to create Constellation on Azure."
     required: false
   sonobuoyTestSuiteCmd:
-    description: 'Which tests should be run? Check README for guidance!'
+    description: "Which tests should be run? Check README for guidance!"
     required: true
+  kubernetesVersion:
+    description: "Kubernetes version to create the cluster from."
+    required: false
   msTeamsWebhook:
-    description: 'WebHook used to notify of failure.'
+    description: "WebHook used to notify of failure."
     required: true
   cosignPublicKey:
-    description: 'Cosign public key to sign measurements.'
+    description: "Cosign public key to sign measurements."
     required: false
   cosignPrivateKey:
-    description: 'Cosign private key to sign measurements.'
+    description: "Cosign private key to sign measurements."
     required: false
   cosignPassword:
-    description: 'Cosign password for private key.'
+    description: "Cosign password for private key."
     required: false
   awsAccessKeyID:
-    description: 'AWS access key ID to upload measurements.'
+    description: "AWS access key ID to upload measurements."
     required: false
   awsSecretAccessKey:
-    description: 'AWS secrets access key to upload measurements.'
+    description: "AWS secrets access key to upload measurements."
     required: false
   awsDefaultRegion:
-    description: 'AWS region of S3 bucket. to upload measurements.'
+    description: "AWS region of S3 bucket. to upload measurements."
     required: false
   awsBucketName:
-    description: 'AWS S3 bucket name to upload measurements.'
+    description: "AWS S3 bucket name to upload measurements."
     required: false
 runs:
-  using: 'composite'
+  using: "composite"
   steps:
     - name: Build CLI
       uses: ./.github/actions/build_cli
@@ -74,6 +77,7 @@ runs:
         workerNodesCount: ${{ inputs.workerNodesCount }}
         controlNodesCount: ${{ inputs.controlNodesCount }}
         machineType: ${{ inputs.machineType }}
+        kubernetesVersion: ${{ inputs.kubernetesVersion }}
     - name: Measure cluster
       uses: ./.github/actions/constellation_measure
       with:

.github/workflows/e2e-test-azure-weekly.yml

@@ -0,0 +1,33 @@
+name: e2e Test Azure Weekly
+# This workflow only runs once per week. It executes the full e2e tests on the latest-1 and latest-2 versions. Currently 1.23 and 1.22.
+# It is executed every sunday at 19:00.
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 19 * * 0"
+jobs:
+  test-older-versions:
+    strategy:
+      matrix:
+        version: ["1.22", "1.23"]
+    runs-on: [self-hosted, edgserver]
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
+      - name: Run Azure E2E test
+        uses: ./.github/actions/e2e_test
+        with:
+          workerNodesCount: "2"
+          controlNodesCount: "1"
+          autoscale: "false"
+          cloudProvider: "azure"
+          machineType: "Standard_D4a_v4"
+          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
+          sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
+          kubernetesVersion: ${{ matrix.version }}
+          msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}

.github/workflows/e2e-test-azure.yml

@@ -3,13 +3,12 @@ name: e2e Test Azure
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 2 * * 2-6'
+    - cron: "0 0 * * 2-6"
 jobs:
-  e2e-test-azure:
+  test-latest:
     runs-on: [self-hosted, edgserver]
     container:
       image: ghcr.io/catthehacker/ubuntu:act-latest
-      options: --privileged
     steps:
       - name: Check out repository
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
@@ -18,11 +17,11 @@ jobs:
       - name: Run Azure E2E test
         uses: ./.github/actions/e2e_test
         with:
-        workerNodesCount: '2'
-        controlNodesCount: '1'
-        autoscale: 'false'
-        cloudProvider: 'azure'
-        machineType: 'Standard_D4a_v4'
+          workerNodesCount: "2"
+          controlNodesCount: "1"
+          autoscale: "false"
+          cloudProvider: "azure"
+          machineType: "Standard_D4a_v4"
           azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
           # TODO: Remove E2E_SKIP once AB#2174 is resolved
           sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
@@ -34,3 +33,28 @@ jobs:
           awsSecretAccessKey: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           awsDefaultRegion: ${{ secrets.AWS_DEFAULT_REGION }}
           awsBucketName: ${{ secrets.PUBLIC_BUCKET_NAME }}
+
+  test-older-versions:
+    strategy:
+      matrix:
+        version: ["1.22", "1.23"]
+    runs-on: [self-hosted, edgserver]
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
+      - name: Run Azure E2E test
+        uses: ./.github/actions/e2e_test
+        with:
+          workerNodesCount: "2"
+          controlNodesCount: "1"
+          autoscale: "false"
+          cloudProvider: "azure"
+          machineType: "Standard_D4a_v4"
+          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
+          sonobuoyTestSuiteCmd: "--mode quick"
+          kubernetesVersion: ${{ matrix.version }}
+          msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}

.github/workflows/e2e-test-gcp-weekly.yml

@@ -0,0 +1,33 @@
+name: e2e Test GCP Weekly
+# This workflow only runs once per week. It executes the full e2e tests on the latest-1 and latest-2 versions. Currently 1.23 and 1.22.
+# It is executed every sunday at 00:00.
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 0"
+jobs:
+  test-older-versions:
+    strategy:
+      matrix:
+        version: ["1.22", "1.23"]
+    runs-on: [self-hosted, edgserver]
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
+      - name: Run GCP E2E test
+        uses: ./.github/actions/e2e_test
+        with:
+          workerNodesCount: "2"
+          controlNodesCount: "1"
+          autoscale: "false"
+          cloudProvider: "gcp"
+          machineType: "n2d-standard-2"
+          gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+          sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
+          kubernetesVersion: ${{ matrix.version }}
+          msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}

.github/workflows/e2e-test-gcp.yml

@@ -3,13 +3,12 @@ name: e2e Test GCP
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 4 * * 2-6'
+    - cron: "0 2 * * 2-6"
 jobs:
-  e2e-test-gcp:
+  test-latest:
     runs-on: [self-hosted, edgserver]
     container:
       image: ghcr.io/catthehacker/ubuntu:act-latest
-      options: --privileged
     steps:
       - name: Check out repository
         uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
@@ -18,11 +17,11 @@ jobs:
       - name: Run GCP E2E test
         uses: ./.github/actions/e2e_test
         with:
-        workerNodesCount: '2'
-        controlNodesCount: '1'
-        autoscale: 'false'
-        cloudProvider: 'gcp'
-        machineType: 'n2d-standard-2'
+          workerNodesCount: "2"
+          controlNodesCount: "1"
+          autoscale: "false"
+          cloudProvider: "gcp"
+          machineType: "n2d-standard-2"
           gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
           # TODO: Remove E2E_SKIP once AB#2174 is resolved
           sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
@@ -34,3 +33,28 @@ jobs:
           awsSecretAccessKey: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           awsDefaultRegion: ${{ secrets.AWS_DEFAULT_REGION }}
           awsBucketName: ${{ secrets.PUBLIC_BUCKET_NAME }}
+
+  test-older-versions:
+    strategy:
+      matrix:
+        version: ["1.22", "1.23"]
+    runs-on: [self-hosted, edgserver]
+    container:
+      image: ghcr.io/catthehacker/ubuntu:act-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
+        with:
+          fetch-depth: 0
+      - name: Run GCP E2E test
+        uses: ./.github/actions/e2e_test
+        with:
+          workerNodesCount: "2"
+          controlNodesCount: "1"
+          autoscale: "false"
+          cloudProvider: "gcp"
+          machineType: "n2d-standard-2"
+          gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
+          sonobuoyTestSuiteCmd: "--mode quick"
+          kubernetesVersion: ${{ matrix.version }}
+          msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}

.github/workflows/e2e-test-manual.yml

@@ -36,11 +36,10 @@ on:
         description: "Which tests should be run? Check README for guidance!"
         default: "--mode quick"
         required: true
-
-# Abort runs of *this* workflow, if a new commit with the same ref is pushed.
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+      kubernetesVersion:
+        description: "Kubernetes version to create the cluster from."
+        default: "1.24"
+        required: true
 
 jobs:
   e2e-test-manual:
@@ -59,4 +58,5 @@ jobs:
           gcp_service_account_json: ${{ secrets.GCP_SERVICE_ACCOUNT }}
           azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
           sonobuoyTestSuiteCmd: ${{ github.event.inputs.sonobuoyTestSuiteCmd }}
+          kubernetesVersion: ${{ github.event.inputs.kubernetesVersion }}
           msTeamsWebhook: ${{ secrets.MS_TEAMS_WEBHOOK_URI }}

CHANGELOG.md

@@ -31,6 +31,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Security
 <!-- In case of vulnerabilities. -->
 ### Internal
+- Run e2e tests on all supported versions.
 
 ## [1.4.0] - 2022-08-02