mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-10-01 01:36:09 -04:00
448 lines
18 KiB
YAML
448 lines
18 KiB
YAML
name: e2e test Terraform provider example
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
ref:
|
|
type: string
|
|
description: "Git ref to checkout"
|
|
cloudProvider:
|
|
description: "Which cloud provider to use."
|
|
type: choice
|
|
options:
|
|
- "aws"
|
|
- "azure"
|
|
- "gcp"
|
|
required: true
|
|
regionZone:
|
|
description: "Region or zone to create the cluster in. Leave empty for default region/zone."
|
|
type: string
|
|
image:
|
|
description: "OS Image version used in the cluster's VMs. If not set, the latest nightly image from main is used."
|
|
type: string
|
|
providerVersion:
|
|
description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source."
|
|
type: string
|
|
toImage:
|
|
description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
|
|
type: string
|
|
required: false
|
|
toKubernetes:
|
|
description: Kubernetes version to target for the upgrade, empty for no upgrade.
|
|
type: string
|
|
required: false
|
|
workflow_call:
|
|
inputs:
|
|
ref:
|
|
type: string
|
|
description: "Git ref to checkout"
|
|
cloudProvider:
|
|
description: "Which cloud provider to use."
|
|
type: string
|
|
required: true
|
|
regionZone:
|
|
description: "Which zone to use."
|
|
type: string
|
|
image:
|
|
description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used."
|
|
type: string
|
|
providerVersion:
|
|
description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source."
|
|
type: string
|
|
toImage:
|
|
description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
|
|
type: string
|
|
required: false
|
|
toKubernetes:
|
|
description: Kubernetes version to target for the upgrade, empty for target's default version.
|
|
type: string
|
|
required: false
|
|
|
|
jobs:
|
|
provider-example-test:
|
|
runs-on: ubuntu-22.04
|
|
permissions:
|
|
id-token: write
|
|
contents: read
|
|
packages: write
|
|
steps:
|
|
- name: Checkout
|
|
id: checkout
|
|
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
|
|
with:
|
|
ref: ${{ inputs.ref || github.head_ref }}
|
|
|
|
- name: Get Latest Image
|
|
id: find-latest-image
|
|
uses: ./.github/actions/find_latest_image
|
|
with:
|
|
git-ref: ${{ inputs.ref }}
|
|
imageVersion: ${{ inputs.image }}
|
|
ref: main
|
|
stream: nightly
|
|
|
|
- name: Upload Terraform module
|
|
uses: ./.github/actions/upload_terraform_module
|
|
|
|
- name: Download Terraform module
|
|
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
|
|
with:
|
|
name: terraform-module
|
|
|
|
- name: Unzip Terraform module
|
|
shell: bash
|
|
run: |
|
|
unzip terraform-module.zip -d ${{ github.workspace }}
|
|
rm terraform-module.zip
|
|
|
|
- name: Create resource prefix
|
|
id: create-prefix
|
|
shell: bash
|
|
run: |
|
|
run_id=${{ github.run_id }}
|
|
last_three="${run_id: -3}"
|
|
echo "prefix=e2e-${last_three}" | tee -a "$GITHUB_OUTPUT"
|
|
|
|
- name: Log in to the Container registry
|
|
uses: ./.github/actions/container_registry_login
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Download CLI # needed to determine K8s version for release versions
|
|
if: inputs.providerVersion != ''
|
|
shell: bash
|
|
run: |
|
|
curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/constellation-linux-amd64
|
|
chmod u+x constellation
|
|
./constellation version
|
|
mkdir -p ${{ github.workspace }}/release
|
|
cp ./constellation ${{ github.workspace }}/release
|
|
|
|
- name: Setup bazel
|
|
uses: ./.github/actions/setup_bazel_nix
|
|
with:
|
|
useCache: "true"
|
|
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
|
|
nixTools: terraform
|
|
|
|
- name: Build Constellation provider and CLI # CLI is needed for the upgrade assert and container push is needed for the microservice upgrade
|
|
working-directory: ${{ github.workspace }}
|
|
id: build
|
|
shell: bash
|
|
run: |
|
|
mkdir build
|
|
cd build
|
|
bazel run //:devbuild --cli_edition=enterprise
|
|
|
|
bazel build //bazel/settings:tag
|
|
repository_root=$(git rev-parse --show-toplevel)
|
|
out_rel=$(bazel cquery --output=files //bazel/settings:tag)
|
|
build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
|
|
echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT"
|
|
|
|
- name: Remove local Terraform registry # otherwise the local registry would be used instead of the public registry
|
|
if: inputs.providerVersion != ''
|
|
shell: bash
|
|
run: |
|
|
bazel build //bazel/settings:tag
|
|
repository_root=$(git rev-parse --show-toplevel)
|
|
out_rel=$(bazel cquery --output=files //bazel/settings:tag)
|
|
build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
|
|
|
|
terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/"
|
|
rm -rf "${terraform_provider_dir}"
|
|
|
|
- name: Login to AWS (IAM + Cluster role)
|
|
if: inputs.cloudProvider == 'aws'
|
|
uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
|
|
with:
|
|
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform
|
|
aws-region: eu-central-1
|
|
# extend token expiry to 6 hours to ensure constellation can terminate
|
|
role-duration-seconds: 21600
|
|
|
|
- name: Login to Azure (IAM + Cluster service principal)
|
|
if: inputs.cloudProvider == 'azure'
|
|
uses: ./.github/actions/login_azure
|
|
with:
|
|
azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }}
|
|
|
|
- name: Login to GCP (IAM + Cluster service account)
|
|
if: inputs.cloudProvider == 'gcp'
|
|
uses: ./.github/actions/login_gcp
|
|
with:
|
|
service_account: "terraform-e2e@constellation-e2e.iam.gserviceaccount.com"
|
|
|
|
- name: Common CSP Terraform overrides
|
|
working-directory: ${{ github.workspace }}
|
|
shell: bash
|
|
run: |
|
|
mkdir cluster
|
|
cd cluster
|
|
if [[ "${{ inputs.providerVersion }}" == "" ]]; then
|
|
prefixed_version=${{ steps.build.outputs.build_version }}
|
|
else
|
|
prefixed_version="${{ inputs.providerVersion }}"
|
|
fi
|
|
version=${prefixed_version#v} # remove v prefix
|
|
|
|
if [[ "${{ inputs.providerVersion }}" == "" ]]; then
|
|
iam_src="../terraform-module/iam/${{ inputs.cloudProvider }}"
|
|
infra_src="../terraform-module/${{ inputs.cloudProvider }}"
|
|
else
|
|
iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/iam/${{ inputs.cloudProvider }}"
|
|
infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/${{ inputs.cloudProvider }}"
|
|
fi
|
|
|
|
# by default use latest nightly image for devbuilds and release image otherwise
|
|
if [[ "${{ inputs.providerVersion }}" == "" ]]; then
|
|
if [[ "${{ inputs.image }}" == "" ]]; then
|
|
image_version="${{ steps.find-latest-image.outputs.image }}"
|
|
else
|
|
image_version="${{ inputs.image }}"
|
|
fi
|
|
else
|
|
if [[ "${{ inputs.image }}" == "" ]]; then
|
|
image_version="${prefixed_version}"
|
|
else
|
|
image_version="${{ inputs.image }}"
|
|
fi
|
|
fi
|
|
|
|
# take the middle (2nd) supported Kubernetes version (default)
|
|
if [[ "${{ inputs.providerVersion }}" != "" ]]; then
|
|
kubernetes_version="$(../release/constellation config kubernetes-versions | awk 'NR==3{print $1}')"
|
|
else
|
|
kubernetes_version="$(../build/constellation config kubernetes-versions | awk 'NR==3{print $1}')"
|
|
fi
|
|
|
|
cat > _override.tf <<EOF
|
|
terraform {
|
|
required_providers {
|
|
constellation = {
|
|
source = "edgelesssys/constellation"
|
|
version = "${version}"
|
|
}
|
|
}
|
|
}
|
|
|
|
locals {
|
|
control_plane_count = 1
|
|
worker_count = 1
|
|
}
|
|
|
|
locals {
|
|
name = "${{ steps.create-prefix.outputs.prefix }}"
|
|
image_version = "${image_version}"
|
|
microservice_version = "${prefixed_version}"
|
|
kubernetes_version = "${kubernetes_version}"
|
|
}
|
|
|
|
module "${{ inputs.cloudProvider }}_iam" {
|
|
source = "${iam_src}"
|
|
}
|
|
|
|
module "${{ inputs.cloudProvider }}_infrastructure" {
|
|
source = "${infra_src}"
|
|
}
|
|
EOF
|
|
cat _override.tf
|
|
|
|
- name: Create GCP Terraform overrides
|
|
if: inputs.cloudProvider == 'gcp'
|
|
working-directory: ${{ github.workspace }}/cluster
|
|
shell: bash
|
|
run: |
|
|
region=$(echo ${{ inputs.regionZone || 'europe-west3-b' }} | rev | cut -c 3- | rev)
|
|
|
|
cat >> _override.tf <<EOF
|
|
locals {
|
|
project_id = "constellation-e2e"
|
|
region = "${region}"
|
|
zone = "${{ inputs.regionZone || 'europe-west3-b' }}"
|
|
}
|
|
EOF
|
|
cat _override.tf
|
|
|
|
- name: Create AWS Terraform overrides
|
|
if: inputs.cloudProvider == 'aws'
|
|
working-directory: ${{ github.workspace }}/cluster
|
|
shell: bash
|
|
run: |
|
|
region=$(echo ${{ inputs.regionZone || 'us-east-2c' }} | rev | cut -c 2- | rev)
|
|
|
|
cat >> _override.tf <<EOF
|
|
locals {
|
|
region = "${region}"
|
|
zone = "${{ inputs.regionZone || 'us-east-2c' }}"
|
|
}
|
|
EOF
|
|
cat _override.tf
|
|
|
|
- name: Copy example Terraform file
|
|
working-directory: ${{ github.workspace }}
|
|
shell: bash
|
|
run: |
|
|
cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ inputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf
|
|
|
|
- name: Apply Terraform Cluster
|
|
id: apply_terraform
|
|
working-directory: ${{ github.workspace }}/cluster
|
|
shell: bash
|
|
run: |
|
|
terraform init
|
|
if [[ "${{ inputs.cloudProvider }}" == "azure" ]]; then
|
|
terraform apply -target module.azure_iam -auto-approve
|
|
terraform apply -target module.azure_infrastructure -auto-approve
|
|
../build/constellation maa-patch "$(terraform output -raw maa_url)"
|
|
terraform apply -target constellation_cluster.azure_example -auto-approve
|
|
else
|
|
terraform apply -auto-approve
|
|
fi
|
|
|
|
- name: Cleanup Terraform Cluster on failure
|
|
# cleanup here already on failure, because the subsequent TF overrides might make the TF config invalid and thus the destroy would fail later
|
|
# outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
|
|
if: failure() && steps.apply_terraform.outcome != 'skipped'
|
|
working-directory: ${{ github.workspace }}/cluster
|
|
shell: bash
|
|
run: |
|
|
terraform init
|
|
terraform destroy -auto-approve
|
|
|
|
- name: Add Provider to local Terraform registry # needed if release version was used before
|
|
if: inputs.providerVersion != ''
|
|
working-directory: ${{ github.workspace }}/build
|
|
shell: bash
|
|
run: |
|
|
bazel run //:devbuild --cli_edition=enterprise
|
|
|
|
- name: Update cluster configuration # for duplicate variable declaration, the last one is used
|
|
working-directory: ${{ github.workspace }}/cluster
|
|
shell: bash
|
|
run: |
|
|
cat >> _override.tf <<EOF
|
|
locals {
|
|
image_version = "${{ inputs.toImage || steps.find-latest-image.outputs.image }}"
|
|
}
|
|
EOF
|
|
|
|
if [[ "${{ inputs.toKubernetes }}" != "" ]]; then
|
|
cat >> _override.tf <<EOF
|
|
resource "constellation_cluster" "${{ inputs.cloudProvider }}_example" {
|
|
kubernetes_version = "${{ inputs.toKubernetes }}"
|
|
}
|
|
EOF
|
|
fi
|
|
|
|
prefixed_version=${{ steps.build.outputs.build_version }}
|
|
version=${prefixed_version#v} # remove v prefix
|
|
|
|
# needs to be explicitly set to upgrade
|
|
cat >> _override.tf <<EOF
|
|
resource "constellation_cluster" "${{ inputs.cloudProvider }}_example" {
|
|
constellation_microservice_version = "${prefixed_version}"
|
|
}
|
|
EOF
|
|
|
|
cat >> _override.tf <<EOF
|
|
terraform {
|
|
required_providers {
|
|
constellation = {
|
|
source = "edgelesssys/constellation"
|
|
version = "${version}"
|
|
}
|
|
}
|
|
}
|
|
EOF
|
|
cat _override.tf
|
|
|
|
- name: Upgrade Terraform Cluster
|
|
working-directory: ${{ github.workspace }}/cluster
|
|
shell: bash
|
|
run: |
|
|
terraform init --upgrade
|
|
terraform apply -auto-approve
|
|
|
|
- name: Assert upgrade successful
|
|
working-directory: ${{ github.workspace }}/cluster
|
|
env:
|
|
IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-latest-image.outputs.image }}
|
|
KUBERNETES: ${{ inputs.toKubernetes }}
|
|
MICROSERVICES: ${{ steps.build.outputs.build_version }}
|
|
WORKERNODES: 1
|
|
CONTROLNODES: 1
|
|
run: |
|
|
terraform output -raw kubeconfig > constellation-admin.conf
|
|
|
|
if [[ -n "${MICROSERVICES}" ]]; then
|
|
MICROSERVICES_FLAG="--target-microservices=${MICROSERVICES}"
|
|
fi
|
|
if [[ -n "${KUBERNETES}" ]]; then
|
|
KUBERNETES_FLAG="--target-kubernetes=${KUBERNETES}"
|
|
fi
|
|
if [[ -n "${IMAGE}" ]]; then
|
|
IMAGE_FLAG="--target-image=${IMAGE}"
|
|
fi
|
|
|
|
# cfg must be in same dir as KUBECONFIG
|
|
../build/constellation config generate "${{ inputs.cloudProvider }}"
|
|
# make cfg valid with fake data
|
|
# IMPORTANT: zone needs to be correct because it is used to resolve the CSP image ref
|
|
if [[ "${{ inputs.cloudProvider }}" == "azure" ]]; then
|
|
location="${{ inputs.regionZone || 'northeurope' }}"
|
|
yq e ".provider.azure.location = \"${location}\"" -i constellation-conf.yaml
|
|
|
|
yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml
|
|
yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml
|
|
yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml
|
|
yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml
|
|
fi
|
|
if [[ "${{ inputs.cloudProvider }}" == "gcp" ]]; then
|
|
zone="${{ inputs.regionZone || 'europe-west3-b' }}"
|
|
region=$(echo "${zone}" | rev | cut -c 2- | rev)
|
|
yq e ".provider.gcp.region = \"${region}\"" -i constellation-conf.yaml
|
|
yq e ".provider.gcp.zone = \"${zone}\"" -i constellation-conf.yaml
|
|
|
|
yq e '.provider.gcp.project = "demo-gcp-project"' -i constellation-conf.yaml
|
|
yq e '.nodeGroups.control_plane_default.zone = "europe-west3-b"' -i constellation-conf.yaml
|
|
# Set the zone for worker_default node group to a fictional value
|
|
yq e '.nodeGroups.worker_default.zone = "europe-west3-b"' -i constellation-conf.yaml
|
|
yq e '.provider.gcp.serviceAccountKeyPath = "/path/to/your/service-account-key.json"' -i constellation-conf.yaml
|
|
fi
|
|
if [[ "${{ inputs.cloudProvider }}" == "aws" ]]; then
|
|
zone=${{ inputs.regionZone || 'us-east-2c' }}
|
|
region=$(echo "${zone}" | rev | cut -c 2- | rev)
|
|
yq e ".provider.aws.region = \"${region}\"" -i constellation-conf.yaml
|
|
yq e ".provider.aws.zone = \"${zone}\"" -i constellation-conf.yaml
|
|
|
|
yq e '.provider.aws.iamProfileControlPlane = "demoControlPlaneIAMProfile"' -i constellation-conf.yaml
|
|
yq e '.provider.aws.iamProfileWorkerNodes = "demoWorkerNodesIAMProfile"' -i constellation-conf.yaml
|
|
yq e '.nodeGroups.control_plane_default.zone = "eu-central-1a"' -i constellation-conf.yaml
|
|
yq e '.nodeGroups.worker_default.zone = "eu-central-1a"' -i constellation-conf.yaml
|
|
fi
|
|
KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --cli "${{ github.workspace }}/build/constellation" "$IMAGE_FLAG" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG"
|
|
|
|
- name: Destroy Terraform Cluster
|
|
# outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
|
|
if: always() && steps.apply_terraform.outcome != 'skipped'
|
|
working-directory: ${{ github.workspace }}/cluster
|
|
shell: bash
|
|
run: |
|
|
terraform init
|
|
terraform destroy -auto-approve
|
|
|
|
- name: Notify about failure
|
|
if: |
|
|
failure() &&
|
|
github.ref == 'refs/heads/main' &&
|
|
github.event_name == 'schedule'
|
|
continue-on-error: true
|
|
uses: ./.github/actions/notify_e2e_failure
|
|
with:
|
|
projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
|
|
test: "terraform-provider-example"
|
|
provider: ${{ inputs.cloudProvider }}
|