constellation/.github/actions/e2e_test/action.yml
Daniel Weiße 724ee44466
ci: Terraform provider e2e tests (#2712)
* Refactor selfManagedInfra input to clusterCreation in e2e tests
* Run e2e test using terraform provider
* Allow insecure measurement fetching in Terraform provider
* Run Terraform provider test instead of module test in weekly runs

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
2023-12-15 10:37:29 +01:00

399 lines
16 KiB
YAML

name: e2e meta test
description: "This test does the infrastructure management and runs the e2e test of your choice."
inputs:
workerNodesCount:
description: "Number of worker nodes to spawn."
default: "2"
controlNodesCount:
description: "Number of control-plane nodes to spawn."
default: "3"
cloudProvider:
description: "Which cloud provider to use."
required: true
machineType:
description: "VM machine type. Make sure it matches selected cloud provider!"
osImage:
description: "OS image to run."
required: true
isDebugImage:
description: "Is OS img a debug img?"
required: true
cliVersion:
description: "Version of a released CLI to download, e.g. 'v2.3.0', leave empty to build it."
kubernetesVersion:
description: "Kubernetes version to create the cluster from."
refStream:
description: "RefStream of the image"
regionZone:
description: "Region or zone to use for resource creation"
required: false
gcpProject:
description: "The GCP project to deploy Constellation in."
required: true
gcpIAMCreateServiceAccount:
description: "Service account with permissions to create IAM configuration on GCP."
required: true
gcpClusterCreateServiceAccount:
description: "Service account with permissions to create a Constellation cluster on GCP."
required: true
awsOpenSearchDomain:
description: "AWS OpenSearch Endpoint Domain to upload the benchmark results."
awsOpenSearchUsers:
description: "AWS OpenSearch User to upload the benchmark results."
awsOpenSearchPwd:
description: "AWS OpenSearch Password to upload the benchmark results."
azureClusterCreateCredentials:
description: "Azure credentials authorized to create a Constellation cluster."
required: true
azureIAMCreateCredentials:
description: "Azure credentials authorized to create an IAM configuration."
required: true
test:
description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, malicious join, nop, upgrade]."
required: true
sonobuoyTestSuiteCmd:
description: "The sonobuoy test suite to run."
buildBuddyApiKey:
description: "BuildBuddy API key for caching Bazel artifacts"
registry:
description: "Container registry to use"
required: true
githubToken:
description: "GitHub authorization token"
required: true
cosignPassword:
description: "The password for the cosign private key. Used for uploading to the config API"
cosignPrivateKey:
description: "The cosign private key. Used for uploading to the config API"
fetchMeasurements:
description: "Update measurements via the 'constellation config fetch-measurements' command."
default: "false"
azureSNPEnforcementPolicy:
description: "Enable security policy for the cluster."
internalLoadBalancer:
description: "Enable internal load balancer for the cluster."
clusterCreation:
description: "How to create infrastructure for the e2e test. One of [cli, self-managed, terraform]."
default: "cli"
s3AccessKey:
description: "Access key for s3proxy"
s3SecretKey:
description: "Secret key for s3proxy"
marketplaceImageVersion:
description: "Marketplace OS image version. Used instead of osImage."
required: false
force:
description: "Set the force-flag on apply to ignore version mismatches."
required: false
outputs:
kubeconfig:
description: "The kubeconfig for the cluster."
value: ${{ steps.constellation-create.outputs.kubeconfig }}
namePrefix:
description: "The name prefix of the cloud resources used in the e2e test."
value: ${{ steps.create-prefix.outputs.prefix }}
runs:
using: "composite"
steps:
- name: Check input
if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "malicious join", "s3proxy", "nop", "upgrade"]'), inputs.test))
shell: bash
run: |
echo "::error::Invalid input for test field: ${{ inputs.test }}"
exit 1
# Perf-bench's network benchmarks require at least two distinct worker nodes.
- name: Validate perf-bench inputs
if: inputs.test == 'perf-bench'
shell: bash
run: |
if [[ "${{ inputs.workerNodesCount }}" -lt 2 ]]; then
echo "::error::Test Perf-Bench requires at least 2 worker nodes."
exit 1
fi
- name: Validate verify input
if: inputs.test == 'verify'
shell: bash
run: |
if [[ "${{ inputs.cosignPassword }}" == '' || "${{ inputs.cosignPrivateKey }}" == '' ]]; then
echo "::error::e2e test verify requires cosignPassword and cosignPrivateKey to be set."
exit 1
fi
- name: Determine build target
id: determine-build-target
shell: bash
run: |
echo "hostOS=$(go env GOOS)" | tee -a "$GITHUB_OUTPUT"
echo "hostArch=$(go env GOARCH)" | tee -a "$GITHUB_OUTPUT"
- name: Setup bazel
uses: ./.github/actions/setup_bazel_nix
with:
useCache: ${{ inputs.buildBuddyApiKey != '' }}
buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
- name: Log in to the Container registry
uses: ./.github/actions/container_registry_login
with:
registry: ${{ inputs.registry }}
username: ${{ github.actor }}
password: ${{ inputs.githubToken }}
- name: Build CLI
if: inputs.cliVersion == ''
uses: ./.github/actions/build_cli
with:
targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
enterpriseCLI: true
outputPath: "build/constellation"
push: ${{ inputs.cliVersion == '' }}
- name: Download CLI
if: inputs.cliVersion != ''
shell: bash
run: |
curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/${{ inputs.cliVersion }}/constellation-linux-amd64
chmod u+x constellation
echo "$(pwd)" >> $GITHUB_PATH
export PATH="$PATH:$(pwd)"
constellation version
# Do not spam license server from pipeline
sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts'
- name: Build Terraform provider binary
if: inputs.clusterCreation == 'terraform' && inputs.cliVersion == ''
uses: ./.github/actions/build_tf_provider
with:
targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
outputPath: "build/terraform-provider-constellation"
- name: Move Terraform provider binary
if: inputs.clusterCreation == 'terraform' && inputs.cliVersion == ''
shell: bash
run: |
bazel build //bazel/settings:tag
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files //bazel/settings:tag)
build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/${{ steps.determine-build-target.outputs.hostOS }}_${{ steps.determine-build-target.outputs.hostArch }}/"
mkdir -p "${terraform_provider_dir}"
mv build/terraform-provider-constellation "${terraform_provider_dir}/terraform-provider-constellation_${build_version}"
- name: Build the bootstrapper
id: build-bootstrapper
if: inputs.isDebugImage == 'true'
uses: ./.github/actions/build_bootstrapper
- name: Build the upgrade-agent
id: build-upgrade-agent
if: inputs.isDebugImage == 'true'
uses: ./.github/actions/build_upgrade_agent
- name: Build cdbg
id: build-cdbg
if: inputs.isDebugImage == 'true'
uses: ./.github/actions/build_cdbg
with:
targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
- name: Login to GCP (IAM service account)
if: inputs.cloudProvider == 'gcp'
uses: ./.github/actions/login_gcp
with:
service_account: ${{ inputs.gcpIAMCreateServiceAccount }}
- name: Login to AWS (IAM role)
if: inputs.cloudProvider == 'aws'
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
with:
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
aws-region: eu-central-1
# extend token expiry to 6 hours to ensure constellation can terminate
role-duration-seconds: 21600
- name: Login to Azure (IAM service principal)
if: inputs.cloudProvider == 'azure'
uses: ./.github/actions/login_azure
with:
azure_credentials: ${{ inputs.azureIAMCreateCredentials }}
- name: Create prefix
id: create-prefix
shell: bash
run: |
uuid=$(uuidgen | tr "[:upper:]" "[:lower:]")
uuid=${uuid%%-*}
echo "uuid=${uuid}" | tee -a $GITHUB_OUTPUT
echo "prefix=e2e-${{ github.run_id }}-${{ github.run_attempt }}-${uuid}" | tee -a $GITHUB_OUTPUT
- name: Pick a random Azure region
id: pick-az-region
uses: ./.github/actions/pick_azure_region
- name: Create IAM configuration
id: constellation-iam-create
uses: ./.github/actions/constellation_iam_create
with:
cloudProvider: ${{ inputs.cloudProvider }}
namePrefix: ${{ steps.create-prefix.outputs.prefix }}
awsZone: ${{ inputs.regionZone || 'us-east-2c' }}
azureRegion: ${{ inputs.regionZone || steps.pick-az-region.outputs.region }}
gcpProjectID: ${{ inputs.gcpProject }}
gcpZone: ${{ inputs.regionZone || 'europe-west3-b' }}
kubernetesVersion: ${{ inputs.kubernetesVersion }}
- name: Login to GCP (Cluster service account)
if: inputs.cloudProvider == 'gcp'
uses: ./.github/actions/login_gcp
with:
service_account: ${{ inputs.gcpClusterCreateServiceAccount }}
- name: Login to AWS (Cluster role)
if: inputs.cloudProvider == 'aws'
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
with:
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster
aws-region: eu-central-1
# extend token expiry to 6 hours to ensure constellation can terminate
role-duration-seconds: 21600
- name: Login to Azure (Cluster service principal)
if: inputs.cloudProvider == 'azure'
uses: ./.github/actions/login_azure
with:
azure_credentials: ${{ inputs.azureClusterCreateCredentials }}
- name: Create cluster
id: constellation-create
uses: ./.github/actions/constellation_create
with:
cloudProvider: ${{ inputs.cloudProvider }}
workerNodesCount: ${{ inputs.workerNodesCount }}
controlNodesCount: ${{ inputs.controlNodesCount }}
machineType: ${{ inputs.machineType }}
osImage: ${{ inputs.osImage }}
isDebugImage: ${{ inputs.isDebugImage }}
artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
fetchMeasurements: ${{ inputs.fetchMeasurements }}
cliVersion: ${{ inputs.cliVersion }}
azureSNPEnforcementPolicy: ${{ inputs.azureSNPEnforcementPolicy }}
azureIAMCreateCredentials: ${{ inputs.azureIAMCreateCredentials }}
azureClusterCreateCredentials: ${{ inputs.azureClusterCreateCredentials }}
kubernetesVersion: ${{ inputs.kubernetesVersion }}
refStream: ${{ inputs.refStream }}
internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
test: ${{ inputs.test }}
clusterCreation: ${{ inputs.clusterCreation }}
marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
force: ${{ inputs.force }}
- name: Deploy log- and metrics-collection (Kubernetes)
id: deploy-logcollection
if: inputs.isDebugImage == 'false'
uses: ./.github/actions/deploy_logcollection
with:
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
opensearchUser: ${{ inputs.awsOpenSearchUsers }}
opensearchPwd: ${{ inputs.awsOpenSearchPwd }}
test: ${{ inputs.test }}
provider: ${{ inputs.cloudProvider }}
isDebugImage: ${{ inputs.isDebugImage }}
kubernetesVersion: ${{ inputs.kubernetesVersion }}
refStream: ${{ inputs.refStream }}
clusterCreation: ${{ inputs.clusterCreation }}
#
# Test payloads
#
- name: Nop test payload
if: (inputs.test == 'nop') || (inputs.test == 'upgrade')
shell: bash
run: |
echo "::warning::This test has a nop payload. It doesn't run any tests."
echo "Sleeping for 30 seconds to allow logs to propagate to the log collection service."
sleep 30
- name: Run sonobuoy quick test
if: inputs.test == 'sonobuoy quick'
uses: ./.github/actions/e2e_sonobuoy
with:
sonobuoyTestSuiteCmd: "--mode quick"
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
- name: Run sonobuoy full test
if: inputs.test == 'sonobuoy full'
uses: ./.github/actions/e2e_sonobuoy
with:
# TODO(3u13r): Remove E2E_SKIP once AB#2174 is resolved
sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
- name: Run autoscaling test
if: inputs.test == 'autoscaling'
uses: ./.github/actions/e2e_autoscaling
with:
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
- name: Run lb test
if: inputs.test == 'lb'
uses: ./.github/actions/e2e_lb
with:
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
- name: Run Performance Benchmark
if: inputs.test == 'perf-bench'
uses: ./.github/actions/e2e_benchmark
with:
cloudProvider: ${{ inputs.cloudProvider }}
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
awsOpenSearchDomain: ${{ inputs.awsOpenSearchDomain }}
awsOpenSearchUsers: ${{ inputs.awsOpenSearchUsers }}
awsOpenSearchPwd: ${{ inputs.awsOpenSearchPwd }}
- name: Run constellation verify test
if: inputs.test == 'verify'
uses: ./.github/actions/e2e_verify
with:
cloudProvider: ${{ inputs.cloudProvider }}
osImage: ${{ steps.constellation-create.outputs.osImageUsed }}
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
cosignPassword: ${{ inputs.cosignPassword }}
cosignPrivateKey: ${{ inputs.cosignPrivateKey }}
- name: Run recover test
if: inputs.test == 'recover'
uses: ./.github/actions/e2e_recover
with:
controlNodesCount: ${{ inputs.controlNodesCount }}
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
- name: Run malicious join test
if: inputs.test == 'malicious join'
uses: ./.github/actions/e2e_malicious_join
with:
cloudProvider: ${{ inputs.cloudProvider }}
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
githubToken: ${{ inputs.githubToken }}
- name: Run s3proxy e2e test
if: inputs.test == 's3proxy'
uses: ./.github/actions/e2e_s3proxy
with:
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
s3AccessKey: ${{ inputs.s3AccessKey }}
s3SecretKey: ${{ inputs.s3SecretKey }}
buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
githubToken: ${{ inputs.githubToken }}