ci: e2e test for Terraform provider examples (#2745)

This commit is contained in:
Adrian Stobbe 2024-01-04 10:00:21 +01:00 committed by GitHub
parent 15cc7b919b
commit 8730e72319
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 340 additions and 38 deletions

View File

@ -281,6 +281,7 @@ runs:
if: inputs.nixTools != '' if: inputs.nixTools != ''
shell: bash shell: bash
env: env:
NIXPKGS_ALLOW_UNFREE: 1
tools: ${{ inputs.nixTools }} tools: ${{ inputs.nixTools }}
repository: ${{ github.repository }} repository: ${{ github.repository }}
gitSha: ${{ github.sha }} gitSha: ${{ github.sha }}

View File

@ -1,10 +1,5 @@
name: Upload Terraform infrastructure module name: Upload Terraform infrastructure module
description: "Upload the Terraform infrastructure module as an artifact." description: "Upload the Terraform infrastructure module as an artifact."
inputs:
encryptionSecret:
description: 'The secret to use for encrypting the artifact.'
required: true
runs: runs:
using: "composite" using: "composite"

View File

@ -175,8 +175,6 @@ jobs:
- name: Upload Terraform infrastructure module - name: Upload Terraform infrastructure module
uses: ./.github/actions/upload_terraform_module uses: ./.github/actions/upload_terraform_module
with:
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
push-containers: push-containers:
runs-on: ubuntu-22.04 runs-on: ubuntu-22.04

View File

@ -0,0 +1,281 @@
name: e2e test Terraform provider example
on:
workflow_dispatch:
inputs:
ref:
type: string
description: "Git ref to checkout"
cloudProvider:
description: "Which cloud provider to use."
type: choice
options:
- "aws"
- "azure"
- "gcp"
required: true
regionZone:
description: "Region or zone to create the cluster in. Leave empty for default region/zone."
type: string
image:
description: "OS Image version used in the cluster's VMs. If not set, the latest nightly image from main is used."
type: string
providerVersion:
description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source."
type: string
workflow_call:
inputs:
ref:
type: string
description: "Git ref to checkout"
cloudProvider:
description: "Which cloud provider to use."
type: string
required: true
regionZone:
description: "Which zone to use."
type: string
image:
description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used."
type: string
providerVersion:
description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source."
type: string
jobs:
provider-example-test:
runs-on: ubuntu-22.04
permissions:
id-token: write
contents: read
packages: write
steps:
- name: Checkout
id: checkout
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
with:
ref: ${{ inputs.ref || github.head_ref }}
- name: Get Latest Image
id: find-latest-image
uses: ./.github/actions/find_latest_image
with:
git-ref: ${{ inputs.ref }}
imageVersion: ${{ inputs.image }}
ref: main
stream: nightly
- name: Upload Terraform module
uses: ./.github/actions/upload_terraform_module
- name: Download Terraform module
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
with:
name: terraform-module
- name: Unzip Terraform module
shell: bash
run: |
unzip terraform-module.zip -d ${{ github.workspace }}
rm terraform-module.zip
- name: Create resource prefix
id: create-prefix
shell: bash
run: |
run_id=${{ github.run_id }}
last_three="${run_id: -3}"
echo "prefix=e2e-${last_three}" | tee -a "$GITHUB_OUTPUT"
- name: Log in to the Container registry
uses: ./.github/actions/container_registry_login
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Setup bazel
uses: ./.github/actions/setup_bazel_nix
with:
useCache: "true"
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
nixTools: terraform
- name: Build Constellation provider and CLI # CLI is needed for the upgrade assert and container push is needed for the microservice upgrade
working-directory: ${{ github.workspace }}
id: build
shell: bash
run: |
mkdir build
cd build
bazel run //:devbuild --cli_edition=enterprise
bazel build //bazel/settings:tag
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files //bazel/settings:tag)
build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT"
- name: Remove local Terraform registry # otherwise the local registry would be used instead of the public registry
if: inputs.providerVersion != ''
shell: bash
run: |
bazel build //bazel/settings:tag
repository_root=$(git rev-parse --show-toplevel)
out_rel=$(bazel cquery --output=files //bazel/settings:tag)
build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/"
rm -rf "${terraform_provider_dir}"
- name: Login to AWS (IAM + Cluster role)
if: inputs.cloudProvider == 'aws'
uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
with:
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform
aws-region: eu-central-1
# extend token expiry to 6 hours to ensure constellation can terminate
role-duration-seconds: 21600
- name: Login to Azure (IAM + Cluster service principal)
if: inputs.cloudProvider == 'azure'
uses: ./.github/actions/login_azure
with:
azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }}
- name: Login to GCP (IAM + Cluster service account)
if: inputs.cloudProvider == 'gcp'
uses: ./.github/actions/login_gcp
with:
service_account: "terraform-e2e@constellation-e2e.iam.gserviceaccount.com"
- name: Common CSP Terraform overrides
working-directory: ${{ github.workspace }}
shell: bash
run: |
mkdir cluster
cd cluster
if [[ "${{ inputs.providerVersion }}" == "" ]]; then
prefixed_version=${{ steps.build.outputs.build_version }}
else
prefixed_version="${{ inputs.providerVersion }}"
fi
version=${prefixed_version#v} # remove v prefix
if [[ "${{ inputs.providerVersion }}" == "" ]]; then
iam_src="../terraform-module/iam/${{ inputs.cloudProvider }}"
infra_src="../terraform-module/${{ inputs.cloudProvider }}"
else
iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/iam/${{ inputs.cloudProvider }}"
infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/${{ inputs.cloudProvider }}"
fi
# by default use latest nightly image for devbuilds and release image otherwise
if [[ "${{ inputs.providerVersion }}" == "" ]]; then
if [[ "${{ inputs.image }}" == "" ]]; then
image_version="${{ steps.find-latest-image.outputs.image }}"
else
image_version="${{ inputs.image }}"
fi
else
if [[ "${{ inputs.image }}" == "" ]]; then
image_version="${prefixed_version}"
else
image_version="${{ inputs.image }}"
fi
fi
cat > _override.tf <<EOF
terraform {
required_providers {
constellation = {
source = "edgelesssys/constellation"
version = "${version}"
}
}
}
locals {
name = "${{ steps.create-prefix.outputs.prefix }}"
version = "${image_version}"
}
module "${{ inputs.cloudProvider }}_iam" {
source = "${iam_src}"
}
module "${{ inputs.cloudProvider }}_infrastructure" {
source = "${infra_src}"
}
EOF
cat _override.tf
- name: Create GCP Terraform overrides
if: inputs.cloudProvider == 'gcp'
working-directory: ${{ github.workspace }}/cluster
shell: bash
run: |
region=$(echo ${{ inputs.regionZone || 'europe-west3-b' }} | rev | cut -c 3- | rev)
cat >> _override.tf <<EOF
locals {
project_id = "constellation-e2e"
region = "${region}"
zone = "${{ inputs.regionZone || 'europe-west3-b' }}"
}
EOF
cat _override.tf
- name: Create AWS Terraform overrides
if: inputs.cloudProvider == 'aws'
working-directory: ${{ github.workspace }}/cluster
shell: bash
run: |
region=$(echo ${{ inputs.regionZone || 'us-east-2c' }} | rev | cut -c 2- | rev)
cat >> _override.tf <<EOF
locals {
region = "${region}"
zone = "${{ inputs.regionZone || 'us-east-2c' }}"
}
EOF
cat _override.tf
- name: Copy example Terraform file
working-directory: ${{ github.workspace }}
shell: bash
run: |
cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ inputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf
- name: Apply Terraform Cluster
id: apply_terraform
working-directory: ${{ github.workspace }}/cluster
shell: bash
run: |
terraform init
if [[ "${{ inputs.cloudProvider }}" == "azure" ]]; then
terraform apply -target module.azure_iam -auto-approve
terraform apply -target module.azure_infrastructure -auto-approve
../build/constellation maa-patch "$(terraform output -raw maa_url)"
TF_LOG=INFO terraform apply -target constellation_cluster.azure_example -auto-approve
else
TF_LOG=INFO terraform apply -auto-approve
fi
- name: Destroy Terraform Cluster
# outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
if: always() && steps.apply_terraform.outcome != 'skipped'
working-directory: ${{ github.workspace }}/cluster
shell: bash
run: |
terraform init
terraform destroy -auto-approve
- name: Notify about failure
if: |
failure() &&
github.ref == 'refs/heads/main' &&
github.event_name == 'schedule'
continue-on-error: true
uses: ./.github/actions/notify_e2e_failure
with:
projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
test: "terraform-provider-example"
provider: ${{ inputs.cloudProvider }}

View File

@ -63,3 +63,4 @@
/tools @malt3 /tools @malt3
/upgrade-agent @3u13r /upgrade-agent @3u13r
/verify @daniel-weisse /verify @daniel-weisse
/terraform-provider-constellation @msanft @elchead

View File

@ -4,8 +4,25 @@ This document explains the basic ways of working with the [Constellation Terrafo
## Building the Terraform Provider ## Building the Terraform Provider
The Constellation Terraform provider can be built through Bazel, either via the [`devbuild` target](./build-develop-deploy.md) (recommended), which will create a `terraform` directory The Constellation Terraform provider can be built through Bazel.
with the provider binary and some utility files in the current working directory, or explicitly via this command:
Use the all-in-one Target (Recommended):
The [`devbuild` target](./build-develop-deploy.md), will create a `terraform` directory
with the provider binary and some utility files in the dedicated local Terraform registry directory.
```bash
bazel run //:devbuild'
```
> [!IMPORTANT] when making changes on the provider without a commit, subsequent applies will fail due to the changed binary hash. To solve this, in your Terraform directory run:
>
> ```bash
> rm .terraform.lock.hcl
> terraform init
> ```
Only build:
```bash ```bash
bazel build //terraform-provider-constellation:tf_provider bazel build //terraform-provider-constellation:tf_provider

View File

@ -2,7 +2,7 @@ terraform {
required_providers { required_providers {
constellation = { constellation = {
source = "edgelesssys/constellation" source = "edgelesssys/constellation"
version = "X.Y.Z" version = "0.0.0" // replace with the version you want to use
} }
random = { random = {
source = "hashicorp/random" source = "hashicorp/random"

View File

@ -2,7 +2,7 @@ terraform {
required_providers { required_providers {
constellation = { constellation = {
source = "edgelesssys/constellation" source = "edgelesssys/constellation"
version = "X.Y.Z" version = "0.0.0" // replace with the version you want to use
} }
random = { random = {
source = "hashicorp/random" source = "hashicorp/random"

View File

@ -2,7 +2,7 @@ terraform {
required_providers { required_providers {
constellation = { constellation = {
source = "edgelesssys/constellation" source = "edgelesssys/constellation"
version = "X.Y.Z" version = "0.0.0" // replace with the version you want to use
} }
random = { random = {
source = "hashicorp/random" source = "hashicorp/random"

View File

@ -103,10 +103,11 @@ type ClusterResourceModel struct {
} }
// networkConfigAttribute is the network config attribute's data model. // networkConfigAttribute is the network config attribute's data model.
// needs basetypes because the struct might be used in ValidateConfig where these values might still be unknown. A go string type cannot handle unknown values.
type networkConfigAttribute struct { type networkConfigAttribute struct {
IPCidrNode string `tfsdk:"ip_cidr_node"` IPCidrNode basetypes.StringValue `tfsdk:"ip_cidr_node"`
IPCidrPod string `tfsdk:"ip_cidr_pod"` IPCidrPod basetypes.StringValue `tfsdk:"ip_cidr_pod"`
IPCidrService string `tfsdk:"ip_cidr_service"` IPCidrService basetypes.StringValue `tfsdk:"ip_cidr_service"`
} }
// gcpAttribute is the gcp attribute's data model. // gcpAttribute is the gcp attribute's data model.
@ -408,26 +409,6 @@ func (r *ClusterResource) ValidateConfig(ctx context.Context, req resource.Valid
"GCP configuration not allowed", "When csp is not set to 'gcp', setting the 'gcp' configuration has no effect.", "GCP configuration not allowed", "When csp is not set to 'gcp', setting the 'gcp' configuration has no effect.",
) )
} }
networkCfg, diags := r.getNetworkConfig(ctx, &data)
resp.Diagnostics.Append(diags...)
if resp.Diagnostics.HasError() {
return
}
// Pod IP CIDR is required for GCP
if strings.EqualFold(data.CSP.ValueString(), cloudprovider.GCP.String()) && networkCfg.IPCidrPod == "" {
resp.Diagnostics.AddAttributeError(
path.Root("network_config").AtName("ip_cidr_pod"),
"Pod IP CIDR missing", "When csp is set to 'gcp', 'ip_cidr_pod' must be set.",
)
}
// Pod IP CIDR should not be set for other CSPs
if !strings.EqualFold(data.CSP.ValueString(), cloudprovider.GCP.String()) && networkCfg.IPCidrPod != "" {
resp.Diagnostics.AddAttributeWarning(
path.Root("network_config").AtName("ip_cidr_pod"),
"Pod IP CIDR not allowed", "When csp is not set to 'gcp', setting 'ip_cidr_pod' has no effect.",
)
}
} }
// Configure configures the resource. // Configure configures the resource.
@ -660,6 +641,29 @@ func (r *ClusterResource) ImportState(ctx context.Context, req resource.ImportSt
resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("master_secret_salt"), masterSecretSalt)...) resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("master_secret_salt"), masterSecretSalt)...)
} }
func (r *ClusterResource) validateGCPNetworkConfig(ctx context.Context, data *ClusterResourceModel) diag.Diagnostics {
networkCfg, diags := r.getNetworkConfig(ctx, data)
if diags.HasError() {
return diags
}
// Pod IP CIDR is required for GCP
if strings.EqualFold(data.CSP.ValueString(), cloudprovider.GCP.String()) && networkCfg.IPCidrPod.ValueString() == "" {
diags.AddAttributeError(
path.Root("network_config").AtName("ip_cidr_pod"),
"Pod IP CIDR missing", "When csp is set to 'gcp', 'ip_cidr_pod' must be set.",
)
}
// Pod IP CIDR should not be set for other CSPs
if !strings.EqualFold(data.CSP.ValueString(), cloudprovider.GCP.String()) && networkCfg.IPCidrPod.ValueString() != "" {
diags.AddAttributeWarning(
path.Root("network_config").AtName("ip_cidr_pod"),
"Pod IP CIDR not allowed", "When csp is not set to 'gcp', setting 'ip_cidr_pod' has no effect.",
)
}
return diags
}
// apply applies changes to a cluster. It can be used for both creating and updating a cluster. // apply applies changes to a cluster. It can be used for both creating and updating a cluster.
// This implements the core part of the Create and Update methods. // This implements the core part of the Create and Update methods.
func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel, skipInitRPC, skipNodeUpgrade bool) diag.Diagnostics { func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel, skipInitRPC, skipNodeUpgrade bool) diag.Diagnostics {
@ -667,6 +671,11 @@ func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel,
// Parse and convert values from the Terraform state // Parse and convert values from the Terraform state
// to formats the Constellation library can work with. // to formats the Constellation library can work with.
convertDiags := r.validateGCPNetworkConfig(ctx, data)
diags.Append(convertDiags...)
if diags.HasError() {
return diags
}
csp := cloudprovider.FromString(data.CSP.ValueString()) csp := cloudprovider.FromString(data.CSP.ValueString())
@ -809,7 +818,7 @@ func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel,
InitSecret: []byte(data.InitSecret.ValueString()), InitSecret: []byte(data.InitSecret.ValueString()),
APIServerCertSANs: apiServerCertSANs, APIServerCertSANs: apiServerCertSANs,
Name: data.Name.ValueString(), Name: data.Name.ValueString(),
IPCidrNode: networkCfg.IPCidrNode, IPCidrNode: networkCfg.IPCidrNode.ValueString(),
}) })
switch csp { switch csp {
case cloudprovider.Azure: case cloudprovider.Azure:
@ -824,7 +833,7 @@ func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel,
case cloudprovider.GCP: case cloudprovider.GCP:
stateFile.Infrastructure.GCP = &state.GCP{ stateFile.Infrastructure.GCP = &state.GCP{
ProjectID: gcpConfig.ProjectID, ProjectID: gcpConfig.ProjectID,
IPCidrPod: networkCfg.IPCidrPod, IPCidrPod: networkCfg.IPCidrPod.ValueString(),
} }
} }
@ -992,7 +1001,7 @@ func (r *ClusterResource) runInitRPC(ctx context.Context, applier *constellation
MeasurementSalt: payload.measurementSalt, MeasurementSalt: payload.measurementSalt,
K8sVersion: payload.k8sVersion, K8sVersion: payload.k8sVersion,
ConformanceMode: false, // Conformance mode does't need to be configurable through the TF provider for now. ConformanceMode: false, // Conformance mode does't need to be configurable through the TF provider for now.
ServiceCIDR: payload.networkCfg.IPCidrService, ServiceCIDR: payload.networkCfg.IPCidrService.ValueString(),
}) })
if err != nil { if err != nil {
var nonRetriable *constellation.NonRetriableInitError var nonRetriable *constellation.NonRetriableInitError