mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-12-11 08:54:21 -05:00
724ee44466
* Refactor selfManagedInfra input to clusterCreation in e2e tests * Run e2e test using terraform provider * Allow insecure measurement fetching in Terraform provider * Run Terraform provider test instead of module test in weekly runs --------- Signed-off-by: Daniel Weiße <dw@edgeless.systems>
399 lines
16 KiB
YAML
399 lines
16 KiB
YAML
name: e2e meta test
|
|
description: "This test does the infrastructure management and runs the e2e test of your choice."
|
|
|
|
inputs:
|
|
workerNodesCount:
|
|
description: "Number of worker nodes to spawn."
|
|
default: "2"
|
|
controlNodesCount:
|
|
description: "Number of control-plane nodes to spawn."
|
|
default: "3"
|
|
cloudProvider:
|
|
description: "Which cloud provider to use."
|
|
required: true
|
|
machineType:
|
|
description: "VM machine type. Make sure it matches selected cloud provider!"
|
|
osImage:
|
|
description: "OS image to run."
|
|
required: true
|
|
isDebugImage:
|
|
description: "Is OS img a debug img?"
|
|
required: true
|
|
cliVersion:
|
|
description: "Version of a released CLI to download, e.g. 'v2.3.0', leave empty to build it."
|
|
kubernetesVersion:
|
|
description: "Kubernetes version to create the cluster from."
|
|
refStream:
|
|
description: "RefStream of the image"
|
|
regionZone:
|
|
description: "Region or zone to use for resource creation"
|
|
required: false
|
|
gcpProject:
|
|
description: "The GCP project to deploy Constellation in."
|
|
required: true
|
|
gcpIAMCreateServiceAccount:
|
|
description: "Service account with permissions to create IAM configuration on GCP."
|
|
required: true
|
|
gcpClusterCreateServiceAccount:
|
|
description: "Service account with permissions to create a Constellation cluster on GCP."
|
|
required: true
|
|
awsOpenSearchDomain:
|
|
description: "AWS OpenSearch Endpoint Domain to upload the benchmark results."
|
|
awsOpenSearchUsers:
|
|
description: "AWS OpenSearch User to upload the benchmark results."
|
|
awsOpenSearchPwd:
|
|
description: "AWS OpenSearch Password to upload the benchmark results."
|
|
azureClusterCreateCredentials:
|
|
description: "Azure credentials authorized to create a Constellation cluster."
|
|
required: true
|
|
azureIAMCreateCredentials:
|
|
description: "Azure credentials authorized to create an IAM configuration."
|
|
required: true
|
|
test:
|
|
description: "The test to run. Can currently be one of [sonobuoy full, sonobuoy quick, autoscaling, lb, perf-bench, verify, recover, malicious join, nop, upgrade]."
|
|
required: true
|
|
sonobuoyTestSuiteCmd:
|
|
description: "The sonobuoy test suite to run."
|
|
buildBuddyApiKey:
|
|
description: "BuildBuddy API key for caching Bazel artifacts"
|
|
registry:
|
|
description: "Container registry to use"
|
|
required: true
|
|
githubToken:
|
|
description: "GitHub authorization token"
|
|
required: true
|
|
cosignPassword:
|
|
description: "The password for the cosign private key. Used for uploading to the config API"
|
|
cosignPrivateKey:
|
|
description: "The cosign private key. Used for uploading to the config API"
|
|
fetchMeasurements:
|
|
description: "Update measurements via the 'constellation config fetch-measurements' command."
|
|
default: "false"
|
|
azureSNPEnforcementPolicy:
|
|
description: "Enable security policy for the cluster."
|
|
internalLoadBalancer:
|
|
description: "Enable internal load balancer for the cluster."
|
|
clusterCreation:
|
|
description: "How to create infrastructure for the e2e test. One of [cli, self-managed, terraform]."
|
|
default: "cli"
|
|
s3AccessKey:
|
|
description: "Access key for s3proxy"
|
|
s3SecretKey:
|
|
description: "Secret key for s3proxy"
|
|
marketplaceImageVersion:
|
|
description: "Marketplace OS image version. Used instead of osImage."
|
|
required: false
|
|
force:
|
|
description: "Set the force-flag on apply to ignore version mismatches."
|
|
required: false
|
|
|
|
outputs:
|
|
kubeconfig:
|
|
description: "The kubeconfig for the cluster."
|
|
value: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
namePrefix:
|
|
description: "The name prefix of the cloud resources used in the e2e test."
|
|
value: ${{ steps.create-prefix.outputs.prefix }}
|
|
|
|
runs:
|
|
using: "composite"
|
|
steps:
|
|
- name: Check input
|
|
if: (!contains(fromJson('["sonobuoy full", "sonobuoy quick", "autoscaling", "perf-bench", "verify", "lb", "recover", "malicious join", "s3proxy", "nop", "upgrade"]'), inputs.test))
|
|
shell: bash
|
|
run: |
|
|
echo "::error::Invalid input for test field: ${{ inputs.test }}"
|
|
exit 1
|
|
|
|
# Perf-bench's network benchmarks require at least two distinct worker nodes.
|
|
- name: Validate perf-bench inputs
|
|
if: inputs.test == 'perf-bench'
|
|
shell: bash
|
|
run: |
|
|
if [[ "${{ inputs.workerNodesCount }}" -lt 2 ]]; then
|
|
echo "::error::Test Perf-Bench requires at least 2 worker nodes."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Validate verify input
|
|
if: inputs.test == 'verify'
|
|
shell: bash
|
|
run: |
|
|
if [[ "${{ inputs.cosignPassword }}" == '' || "${{ inputs.cosignPrivateKey }}" == '' ]]; then
|
|
echo "::error::e2e test verify requires cosignPassword and cosignPrivateKey to be set."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Determine build target
|
|
id: determine-build-target
|
|
shell: bash
|
|
run: |
|
|
echo "hostOS=$(go env GOOS)" | tee -a "$GITHUB_OUTPUT"
|
|
echo "hostArch=$(go env GOARCH)" | tee -a "$GITHUB_OUTPUT"
|
|
|
|
- name: Setup bazel
|
|
uses: ./.github/actions/setup_bazel_nix
|
|
with:
|
|
useCache: ${{ inputs.buildBuddyApiKey != '' }}
|
|
buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
|
|
|
|
- name: Log in to the Container registry
|
|
uses: ./.github/actions/container_registry_login
|
|
with:
|
|
registry: ${{ inputs.registry }}
|
|
username: ${{ github.actor }}
|
|
password: ${{ inputs.githubToken }}
|
|
|
|
- name: Build CLI
|
|
if: inputs.cliVersion == ''
|
|
uses: ./.github/actions/build_cli
|
|
with:
|
|
targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
|
|
targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
|
|
enterpriseCLI: true
|
|
outputPath: "build/constellation"
|
|
push: ${{ inputs.cliVersion == '' }}
|
|
|
|
- name: Download CLI
|
|
if: inputs.cliVersion != ''
|
|
shell: bash
|
|
run: |
|
|
curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/${{ inputs.cliVersion }}/constellation-linux-amd64
|
|
chmod u+x constellation
|
|
echo "$(pwd)" >> $GITHUB_PATH
|
|
export PATH="$PATH:$(pwd)"
|
|
constellation version
|
|
# Do not spam license server from pipeline
|
|
sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts'
|
|
|
|
- name: Build Terraform provider binary
|
|
if: inputs.clusterCreation == 'terraform' && inputs.cliVersion == ''
|
|
uses: ./.github/actions/build_tf_provider
|
|
with:
|
|
targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
|
|
targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
|
|
outputPath: "build/terraform-provider-constellation"
|
|
|
|
- name: Move Terraform provider binary
|
|
if: inputs.clusterCreation == 'terraform' && inputs.cliVersion == ''
|
|
shell: bash
|
|
run: |
|
|
bazel build //bazel/settings:tag
|
|
|
|
repository_root=$(git rev-parse --show-toplevel)
|
|
out_rel=$(bazel cquery --output=files //bazel/settings:tag)
|
|
build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
|
|
|
|
terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/${{ steps.determine-build-target.outputs.hostOS }}_${{ steps.determine-build-target.outputs.hostArch }}/"
|
|
mkdir -p "${terraform_provider_dir}"
|
|
mv build/terraform-provider-constellation "${terraform_provider_dir}/terraform-provider-constellation_${build_version}"
|
|
|
|
- name: Build the bootstrapper
|
|
id: build-bootstrapper
|
|
if: inputs.isDebugImage == 'true'
|
|
uses: ./.github/actions/build_bootstrapper
|
|
|
|
- name: Build the upgrade-agent
|
|
id: build-upgrade-agent
|
|
if: inputs.isDebugImage == 'true'
|
|
uses: ./.github/actions/build_upgrade_agent
|
|
|
|
- name: Build cdbg
|
|
id: build-cdbg
|
|
if: inputs.isDebugImage == 'true'
|
|
uses: ./.github/actions/build_cdbg
|
|
with:
|
|
targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
|
|
targetArch: ${{ steps.determine-build-target.outputs.hostArch }}
|
|
|
|
- name: Login to GCP (IAM service account)
|
|
if: inputs.cloudProvider == 'gcp'
|
|
uses: ./.github/actions/login_gcp
|
|
with:
|
|
service_account: ${{ inputs.gcpIAMCreateServiceAccount }}
|
|
|
|
- name: Login to AWS (IAM role)
|
|
if: inputs.cloudProvider == 'aws'
|
|
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
|
|
with:
|
|
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
|
|
aws-region: eu-central-1
|
|
# extend token expiry to 6 hours to ensure constellation can terminate
|
|
role-duration-seconds: 21600
|
|
|
|
- name: Login to Azure (IAM service principal)
|
|
if: inputs.cloudProvider == 'azure'
|
|
uses: ./.github/actions/login_azure
|
|
with:
|
|
azure_credentials: ${{ inputs.azureIAMCreateCredentials }}
|
|
|
|
- name: Create prefix
|
|
id: create-prefix
|
|
shell: bash
|
|
run: |
|
|
uuid=$(uuidgen | tr "[:upper:]" "[:lower:]")
|
|
uuid=${uuid%%-*}
|
|
echo "uuid=${uuid}" | tee -a $GITHUB_OUTPUT
|
|
echo "prefix=e2e-${{ github.run_id }}-${{ github.run_attempt }}-${uuid}" | tee -a $GITHUB_OUTPUT
|
|
|
|
- name: Pick a random Azure region
|
|
id: pick-az-region
|
|
uses: ./.github/actions/pick_azure_region
|
|
|
|
- name: Create IAM configuration
|
|
id: constellation-iam-create
|
|
uses: ./.github/actions/constellation_iam_create
|
|
with:
|
|
cloudProvider: ${{ inputs.cloudProvider }}
|
|
namePrefix: ${{ steps.create-prefix.outputs.prefix }}
|
|
awsZone: ${{ inputs.regionZone || 'us-east-2c' }}
|
|
azureRegion: ${{ inputs.regionZone || steps.pick-az-region.outputs.region }}
|
|
gcpProjectID: ${{ inputs.gcpProject }}
|
|
gcpZone: ${{ inputs.regionZone || 'europe-west3-b' }}
|
|
kubernetesVersion: ${{ inputs.kubernetesVersion }}
|
|
|
|
- name: Login to GCP (Cluster service account)
|
|
if: inputs.cloudProvider == 'gcp'
|
|
uses: ./.github/actions/login_gcp
|
|
with:
|
|
service_account: ${{ inputs.gcpClusterCreateServiceAccount }}
|
|
|
|
- name: Login to AWS (Cluster role)
|
|
if: inputs.cloudProvider == 'aws'
|
|
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
|
|
with:
|
|
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster
|
|
aws-region: eu-central-1
|
|
# extend token expiry to 6 hours to ensure constellation can terminate
|
|
role-duration-seconds: 21600
|
|
|
|
- name: Login to Azure (Cluster service principal)
|
|
if: inputs.cloudProvider == 'azure'
|
|
uses: ./.github/actions/login_azure
|
|
with:
|
|
azure_credentials: ${{ inputs.azureClusterCreateCredentials }}
|
|
|
|
- name: Create cluster
|
|
id: constellation-create
|
|
uses: ./.github/actions/constellation_create
|
|
with:
|
|
cloudProvider: ${{ inputs.cloudProvider }}
|
|
workerNodesCount: ${{ inputs.workerNodesCount }}
|
|
controlNodesCount: ${{ inputs.controlNodesCount }}
|
|
machineType: ${{ inputs.machineType }}
|
|
osImage: ${{ inputs.osImage }}
|
|
isDebugImage: ${{ inputs.isDebugImage }}
|
|
artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
|
|
fetchMeasurements: ${{ inputs.fetchMeasurements }}
|
|
cliVersion: ${{ inputs.cliVersion }}
|
|
azureSNPEnforcementPolicy: ${{ inputs.azureSNPEnforcementPolicy }}
|
|
azureIAMCreateCredentials: ${{ inputs.azureIAMCreateCredentials }}
|
|
azureClusterCreateCredentials: ${{ inputs.azureClusterCreateCredentials }}
|
|
kubernetesVersion: ${{ inputs.kubernetesVersion }}
|
|
refStream: ${{ inputs.refStream }}
|
|
internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
|
|
test: ${{ inputs.test }}
|
|
clusterCreation: ${{ inputs.clusterCreation }}
|
|
marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
|
|
force: ${{ inputs.force }}
|
|
|
|
- name: Deploy log- and metrics-collection (Kubernetes)
|
|
id: deploy-logcollection
|
|
if: inputs.isDebugImage == 'false'
|
|
uses: ./.github/actions/deploy_logcollection
|
|
with:
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
opensearchUser: ${{ inputs.awsOpenSearchUsers }}
|
|
opensearchPwd: ${{ inputs.awsOpenSearchPwd }}
|
|
test: ${{ inputs.test }}
|
|
provider: ${{ inputs.cloudProvider }}
|
|
isDebugImage: ${{ inputs.isDebugImage }}
|
|
kubernetesVersion: ${{ inputs.kubernetesVersion }}
|
|
refStream: ${{ inputs.refStream }}
|
|
clusterCreation: ${{ inputs.clusterCreation }}
|
|
|
|
#
|
|
# Test payloads
|
|
#
|
|
- name: Nop test payload
|
|
if: (inputs.test == 'nop') || (inputs.test == 'upgrade')
|
|
shell: bash
|
|
run: |
|
|
echo "::warning::This test has a nop payload. It doesn't run any tests."
|
|
echo "Sleeping for 30 seconds to allow logs to propagate to the log collection service."
|
|
sleep 30
|
|
|
|
- name: Run sonobuoy quick test
|
|
if: inputs.test == 'sonobuoy quick'
|
|
uses: ./.github/actions/e2e_sonobuoy
|
|
with:
|
|
sonobuoyTestSuiteCmd: "--mode quick"
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
|
|
|
|
- name: Run sonobuoy full test
|
|
if: inputs.test == 'sonobuoy full'
|
|
uses: ./.github/actions/e2e_sonobuoy
|
|
with:
|
|
# TODO(3u13r): Remove E2E_SKIP once AB#2174 is resolved
|
|
sonobuoyTestSuiteCmd: '--plugin e2e --plugin-env e2e.E2E_FOCUS="\[Conformance\]" --plugin-env e2e.E2E_SKIP="for service with type clusterIP|HostPort validates that there is no conflict between pods with same hostPort but different hostIP and protocol" --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-plugin.yaml --plugin https://raw.githubusercontent.com/vmware-tanzu/sonobuoy-plugins/master/cis-benchmarks/kube-bench-master-plugin.yaml'
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
artifactNameSuffix: ${{ steps.create-prefix.outputs.prefix }}
|
|
|
|
- name: Run autoscaling test
|
|
if: inputs.test == 'autoscaling'
|
|
uses: ./.github/actions/e2e_autoscaling
|
|
with:
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
|
|
- name: Run lb test
|
|
if: inputs.test == 'lb'
|
|
uses: ./.github/actions/e2e_lb
|
|
with:
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
|
|
- name: Run Performance Benchmark
|
|
if: inputs.test == 'perf-bench'
|
|
uses: ./.github/actions/e2e_benchmark
|
|
with:
|
|
cloudProvider: ${{ inputs.cloudProvider }}
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
awsOpenSearchDomain: ${{ inputs.awsOpenSearchDomain }}
|
|
awsOpenSearchUsers: ${{ inputs.awsOpenSearchUsers }}
|
|
awsOpenSearchPwd: ${{ inputs.awsOpenSearchPwd }}
|
|
|
|
- name: Run constellation verify test
|
|
if: inputs.test == 'verify'
|
|
uses: ./.github/actions/e2e_verify
|
|
with:
|
|
cloudProvider: ${{ inputs.cloudProvider }}
|
|
osImage: ${{ steps.constellation-create.outputs.osImageUsed }}
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
cosignPassword: ${{ inputs.cosignPassword }}
|
|
cosignPrivateKey: ${{ inputs.cosignPrivateKey }}
|
|
|
|
- name: Run recover test
|
|
if: inputs.test == 'recover'
|
|
uses: ./.github/actions/e2e_recover
|
|
with:
|
|
controlNodesCount: ${{ inputs.controlNodesCount }}
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
|
|
- name: Run malicious join test
|
|
if: inputs.test == 'malicious join'
|
|
uses: ./.github/actions/e2e_malicious_join
|
|
with:
|
|
cloudProvider: ${{ inputs.cloudProvider }}
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
githubToken: ${{ inputs.githubToken }}
|
|
|
|
- name: Run s3proxy e2e test
|
|
if: inputs.test == 's3proxy'
|
|
uses: ./.github/actions/e2e_s3proxy
|
|
with:
|
|
kubeconfig: ${{ steps.constellation-create.outputs.kubeconfig }}
|
|
s3AccessKey: ${{ inputs.s3AccessKey }}
|
|
s3SecretKey: ${{ inputs.s3SecretKey }}
|
|
buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
|
|
githubToken: ${{ inputs.githubToken }}
|