ci: e2e test for Terraform provider examples (#2745)

.github/actions/setup_bazel_nix/action.yml

@@ -281,6 +281,7 @@ runs:
       if: inputs.nixTools != ''
       shell: bash
       env:
+        NIXPKGS_ALLOW_UNFREE: 1
         tools: ${{ inputs.nixTools }}
         repository: ${{ github.repository }}
         gitSha: ${{ github.sha }}

.github/actions/upload_terraform_module/action.yml

@@ -1,10 +1,5 @@
 name: Upload Terraform infrastructure module
 description: "Upload the Terraform infrastructure module as an artifact."
-inputs:
-  encryptionSecret:
-      description: 'The secret to use for encrypting the artifact.'
-      required: true
-
 
 runs:
   using: "composite"

.github/workflows/draft-release.yml

@@ -175,8 +175,6 @@ jobs:
 
       - name: Upload Terraform infrastructure module
         uses: ./.github/actions/upload_terraform_module
-        with:
-          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 
   push-containers:
     runs-on: ubuntu-22.04

.github/workflows/e2e-test-provider-example.yml

@@ -0,0 +1,281 @@
+name: e2e test Terraform provider example
+
+on:
+  workflow_dispatch:
+    inputs:
+      ref:
+        type: string
+        description: "Git ref to checkout"
+      cloudProvider:
+        description: "Which cloud provider to use."
+        type: choice
+        options:
+          - "aws"
+          - "azure"
+          - "gcp"
+        required: true
+      regionZone:
+        description: "Region or zone to create the cluster in. Leave empty for default region/zone."
+        type: string
+      image:
+        description: "OS Image version used in the cluster's VMs. If not set, the latest nightly image from main is used."
+        type: string
+      providerVersion:
+        description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source."
+        type: string
+  workflow_call:
+    inputs:
+      ref:
+        type: string
+        description: "Git ref to checkout"
+      cloudProvider:
+        description: "Which cloud provider to use."
+        type: string
+        required: true
+      regionZone:
+        description: "Which zone to use."
+        type: string
+      image:
+        description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used."
+        type: string
+      providerVersion:
+        description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source."
+        type: string
+
+jobs:
+  provider-example-test:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+        with:
+          ref: ${{ inputs.ref || github.head_ref }}
+
+      - name: Get Latest Image
+        id: find-latest-image
+        uses: ./.github/actions/find_latest_image
+        with:
+          git-ref: ${{ inputs.ref }}
+          imageVersion: ${{ inputs.image }}
+          ref: main
+          stream: nightly
+
+      - name: Upload Terraform module
+        uses: ./.github/actions/upload_terraform_module
+
+      - name: Download Terraform module
+        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        with:
+          name: terraform-module
+
+      - name: Unzip Terraform module
+        shell: bash
+        run: |
+          unzip terraform-module.zip -d ${{ github.workspace }}
+          rm terraform-module.zip
+
+      - name: Create resource prefix
+        id: create-prefix
+        shell: bash
+        run: |
+          run_id=${{ github.run_id }}
+          last_three="${run_id: -3}"
+          echo "prefix=e2e-${last_three}" | tee -a "$GITHUB_OUTPUT"
+
+      - name: Log in to the Container registry
+        uses: ./.github/actions/container_registry_login
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup bazel
+        uses: ./.github/actions/setup_bazel_nix
+        with:
+          useCache: "true"
+          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+          nixTools: terraform
+
+      - name: Build Constellation provider and CLI # CLI is needed for the upgrade assert and container push is needed for the microservice upgrade
+        working-directory: ${{ github.workspace }}
+        id: build
+        shell: bash
+        run: |
+          mkdir build
+          cd build
+          bazel run //:devbuild --cli_edition=enterprise
+
+          bazel build //bazel/settings:tag
+          repository_root=$(git rev-parse --show-toplevel)
+          out_rel=$(bazel cquery --output=files //bazel/settings:tag)
+          build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
+          echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT"
+
+      - name: Remove local Terraform registry # otherwise the local registry would be used instead of the public registry
+        if: inputs.providerVersion != ''
+        shell: bash
+        run: |
+          bazel build //bazel/settings:tag
+          repository_root=$(git rev-parse --show-toplevel)
+          out_rel=$(bazel cquery --output=files //bazel/settings:tag)
+          build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
+
+          terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/"
+          rm -rf "${terraform_provider_dir}"
+
+      - name: Login to AWS (IAM + Cluster role)
+        if: inputs.cloudProvider == 'aws'
+        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform
+          aws-region: eu-central-1
+          # extend token expiry to 6 hours to ensure constellation can terminate
+          role-duration-seconds: 21600
+
+      - name: Login to Azure (IAM + Cluster service principal)
+        if: inputs.cloudProvider == 'azure'
+        uses: ./.github/actions/login_azure
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }}
+
+      - name: Login to GCP (IAM + Cluster service account)
+        if: inputs.cloudProvider == 'gcp'
+        uses: ./.github/actions/login_gcp
+        with:
+          service_account: "terraform-e2e@constellation-e2e.iam.gserviceaccount.com"
+
+      - name: Common CSP Terraform overrides
+        working-directory: ${{ github.workspace }}
+        shell: bash
+        run: |
+          mkdir cluster
+          cd cluster
+          if [[ "${{ inputs.providerVersion }}" == "" ]]; then
+            prefixed_version=${{ steps.build.outputs.build_version }}
+          else
+            prefixed_version="${{ inputs.providerVersion }}"
+          fi
+          version=${prefixed_version#v} # remove v prefix
+
+          if [[ "${{ inputs.providerVersion }}" == "" ]]; then
+            iam_src="../terraform-module/iam/${{ inputs.cloudProvider }}"
+            infra_src="../terraform-module/${{ inputs.cloudProvider }}"
+          else
+            iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/iam/${{ inputs.cloudProvider }}"
+            infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/${{ inputs.cloudProvider }}"
+          fi
+
+          # by default use latest nightly image for devbuilds and release image otherwise
+          if [[ "${{ inputs.providerVersion }}" == "" ]]; then
+            if [[ "${{ inputs.image }}" == "" ]]; then
+              image_version="${{ steps.find-latest-image.outputs.image }}"
+            else
+              image_version="${{ inputs.image }}"
+            fi
+          else
+            if [[ "${{ inputs.image }}" == "" ]]; then
+              image_version="${prefixed_version}"
+            else
+              image_version="${{ inputs.image }}"
+            fi
+          fi
+
+          cat > _override.tf <<EOF
+          terraform {
+            required_providers {
+              constellation = {
+                source  = "edgelesssys/constellation"
+                version            = "${version}"
+              }
+            }
+          }
+          locals {
+            name                = "${{ steps.create-prefix.outputs.prefix }}"
+            version            = "${image_version}"
+          }
+          module "${{ inputs.cloudProvider }}_iam" {
+            source = "${iam_src}"
+          }
+          module "${{ inputs.cloudProvider }}_infrastructure" {
+            source = "${infra_src}"
+          }
+          EOF
+          cat _override.tf
+
+      - name: Create GCP Terraform overrides
+        if: inputs.cloudProvider == 'gcp'
+        working-directory: ${{ github.workspace }}/cluster
+        shell: bash
+        run: |
+          region=$(echo ${{ inputs.regionZone || 'europe-west3-b' }} | rev | cut -c 3- | rev)
+
+          cat >> _override.tf <<EOF
+          locals {
+            project_id         = "constellation-e2e"
+            region = "${region}"
+            zone = "${{ inputs.regionZone || 'europe-west3-b' }}"
+          }
+          EOF
+          cat _override.tf
+
+      - name: Create AWS Terraform overrides
+        if: inputs.cloudProvider == 'aws'
+        working-directory: ${{ github.workspace }}/cluster
+        shell: bash
+        run: |
+          region=$(echo ${{ inputs.regionZone || 'us-east-2c' }} | rev | cut -c 2- | rev)
+
+          cat >> _override.tf <<EOF
+          locals {
+            region = "${region}"
+            zone = "${{ inputs.regionZone || 'us-east-2c' }}"
+          }
+          EOF
+          cat _override.tf
+
+      - name: Copy example Terraform file
+        working-directory: ${{ github.workspace }}
+        shell: bash
+        run: |
+          cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ inputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf
+
+      - name: Apply Terraform Cluster
+        id: apply_terraform
+        working-directory: ${{ github.workspace }}/cluster
+        shell: bash
+        run: |
+          terraform init
+          if [[ "${{ inputs.cloudProvider }}" == "azure" ]]; then
+            terraform apply -target module.azure_iam -auto-approve
+            terraform apply -target module.azure_infrastructure -auto-approve
+            ../build/constellation maa-patch "$(terraform output -raw maa_url)"
+            TF_LOG=INFO terraform apply -target constellation_cluster.azure_example -auto-approve
+          else
+            TF_LOG=INFO terraform apply -auto-approve
+          fi
+
+      - name: Destroy Terraform Cluster
+      # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
+        if: always() && steps.apply_terraform.outcome != 'skipped'
+        working-directory: ${{ github.workspace }}/cluster
+        shell: bash
+        run: |
+          terraform init
+          terraform destroy -auto-approve
+
+      - name: Notify about failure
+        if: |
+          failure() &&
+          github.ref == 'refs/heads/main' &&
+          github.event_name == 'schedule'
+        continue-on-error: true
+        uses: ./.github/actions/notify_e2e_failure
+        with:
+          projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
+          test: "terraform-provider-example"
+          provider: ${{ inputs.cloudProvider }}

CODEOWNERS

@@ -63,3 +63,4 @@
 /tools @malt3
 /upgrade-agent @3u13r
 /verify @daniel-weisse
+/terraform-provider-constellation @msanft @elchead

dev-docs/workflows/terraform-provider.md

@@ -4,8 +4,25 @@ This document explains the basic ways of working with the [Constellation Terrafo
 
 ## Building the Terraform Provider
 
-The Constellation Terraform provider can be built through Bazel, either via the [`devbuild` target](./build-develop-deploy.md) (recommended), which will create a `terraform` directory
+The Constellation Terraform provider can be built through Bazel.
-with the provider binary and some utility files in the current working directory, or explicitly via this command:
+
+Use the all-in-one Target (Recommended):
+
+The [`devbuild` target](./build-develop-deploy.md), will create a `terraform` directory
+with the provider binary and some utility files in the dedicated local Terraform registry directory.
+
+```bash
+bazel run //:devbuild'
+```
+
+> [!IMPORTANT] when making changes on the provider without a commit, subsequent applies will fail due to the changed binary hash. To solve this, in your Terraform directory run:
+>
+> ```bash
+> rm .terraform.lock.hcl
+> terraform init
+> ```
+
+Only build:
 
 ```bash
 bazel build //terraform-provider-constellation:tf_provider

terraform-provider-constellation/examples/full/aws/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     constellation = {
       source  = "edgelesssys/constellation"
-      version = "X.Y.Z"
+      version = "0.0.0" // replace with the version you want to use
     }
     random = {
       source  = "hashicorp/random"

terraform-provider-constellation/examples/full/azure/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     constellation = {
       source  = "edgelesssys/constellation"
-      version = "X.Y.Z"
+      version = "0.0.0" // replace with the version you want to use
     }
     random = {
       source  = "hashicorp/random"

terraform-provider-constellation/examples/full/gcp/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     constellation = {
       source  = "edgelesssys/constellation"
-      version = "X.Y.Z"
+      version = "0.0.0" // replace with the version you want to use
     }
     random = {
       source  = "hashicorp/random"

terraform-provider-constellation/internal/provider/cluster_resource.go

@@ -103,10 +103,11 @@ type ClusterResourceModel struct {
 }
 
 // networkConfigAttribute is the network config attribute's data model.
+// needs basetypes because the struct might be used in ValidateConfig where these values might still be unknown. A go string type cannot handle unknown values.
 type networkConfigAttribute struct {
-	IPCidrNode    string `tfsdk:"ip_cidr_node"`
+	IPCidrNode    basetypes.StringValue `tfsdk:"ip_cidr_node"`
-	IPCidrPod     string `tfsdk:"ip_cidr_pod"`
+	IPCidrPod     basetypes.StringValue `tfsdk:"ip_cidr_pod"`
-	IPCidrService string `tfsdk:"ip_cidr_service"`
+	IPCidrService basetypes.StringValue `tfsdk:"ip_cidr_service"`
 }
 
 // gcpAttribute is the gcp attribute's data model.
@@ -408,26 +409,6 @@ func (r *ClusterResource) ValidateConfig(ctx context.Context, req resource.Valid
 			"GCP configuration not allowed", "When csp is not set to 'gcp', setting the 'gcp' configuration has no effect.",
 		)
 	}
-
-	networkCfg, diags := r.getNetworkConfig(ctx, &data)
-	resp.Diagnostics.Append(diags...)
-	if resp.Diagnostics.HasError() {
-		return
-	}
-	// Pod IP CIDR is required for GCP
-	if strings.EqualFold(data.CSP.ValueString(), cloudprovider.GCP.String()) && networkCfg.IPCidrPod == "" {
-		resp.Diagnostics.AddAttributeError(
-			path.Root("network_config").AtName("ip_cidr_pod"),
-			"Pod IP CIDR missing", "When csp is set to 'gcp', 'ip_cidr_pod' must be set.",
-		)
-	}
-	// Pod IP CIDR should not be set for other CSPs
-	if !strings.EqualFold(data.CSP.ValueString(), cloudprovider.GCP.String()) && networkCfg.IPCidrPod != "" {
-		resp.Diagnostics.AddAttributeWarning(
-			path.Root("network_config").AtName("ip_cidr_pod"),
-			"Pod IP CIDR not allowed", "When csp is not set to 'gcp', setting 'ip_cidr_pod' has no effect.",
-		)
-	}
 }
 
 // Configure configures the resource.
@@ -660,6 +641,29 @@ func (r *ClusterResource) ImportState(ctx context.Context, req resource.ImportSt
 	resp.Diagnostics.Append(resp.State.SetAttribute(ctx, path.Root("master_secret_salt"), masterSecretSalt)...)
 }
 
+func (r *ClusterResource) validateGCPNetworkConfig(ctx context.Context, data *ClusterResourceModel) diag.Diagnostics {
+	networkCfg, diags := r.getNetworkConfig(ctx, data)
+	if diags.HasError() {
+		return diags
+	}
+
+	// Pod IP CIDR is required for GCP
+	if strings.EqualFold(data.CSP.ValueString(), cloudprovider.GCP.String()) && networkCfg.IPCidrPod.ValueString() == "" {
+		diags.AddAttributeError(
+			path.Root("network_config").AtName("ip_cidr_pod"),
+			"Pod IP CIDR missing", "When csp is set to 'gcp', 'ip_cidr_pod' must be set.",
+		)
+	}
+	// Pod IP CIDR should not be set for other CSPs
+	if !strings.EqualFold(data.CSP.ValueString(), cloudprovider.GCP.String()) && networkCfg.IPCidrPod.ValueString() != "" {
+		diags.AddAttributeWarning(
+			path.Root("network_config").AtName("ip_cidr_pod"),
+			"Pod IP CIDR not allowed", "When csp is not set to 'gcp', setting 'ip_cidr_pod' has no effect.",
+		)
+	}
+	return diags
+}
+
 // apply applies changes to a cluster. It can be used for both creating and updating a cluster.
 // This implements the core part of the Create and Update methods.
 func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel, skipInitRPC, skipNodeUpgrade bool) diag.Diagnostics {
@@ -667,6 +671,11 @@ func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel,
 
 	// Parse and convert values from the Terraform state
 	// to formats the Constellation library can work with.
+	convertDiags := r.validateGCPNetworkConfig(ctx, data)
+	diags.Append(convertDiags...)
+	if diags.HasError() {
+		return diags
+	}
 
 	csp := cloudprovider.FromString(data.CSP.ValueString())
 
@@ -809,7 +818,7 @@ func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel,
 		InitSecret:        []byte(data.InitSecret.ValueString()),
 		APIServerCertSANs: apiServerCertSANs,
 		Name:              data.Name.ValueString(),
-		IPCidrNode:        networkCfg.IPCidrNode,
+		IPCidrNode:        networkCfg.IPCidrNode.ValueString(),
 	})
 	switch csp {
 	case cloudprovider.Azure:
@@ -824,7 +833,7 @@ func (r *ClusterResource) apply(ctx context.Context, data *ClusterResourceModel,
 	case cloudprovider.GCP:
 		stateFile.Infrastructure.GCP = &state.GCP{
 			ProjectID: gcpConfig.ProjectID,
-			IPCidrPod: networkCfg.IPCidrPod,
+			IPCidrPod: networkCfg.IPCidrPod.ValueString(),
 		}
 	}
 
@@ -992,7 +1001,7 @@ func (r *ClusterResource) runInitRPC(ctx context.Context, applier *constellation
 			MeasurementSalt: payload.measurementSalt,
 			K8sVersion:      payload.k8sVersion,
 			ConformanceMode: false, // Conformance mode does't need to be configurable through the TF provider for now.
-			ServiceCIDR:     payload.networkCfg.IPCidrService,
+			ServiceCIDR:     payload.networkCfg.IPCidrService.ValueString(),
 		})
 	if err != nil {
 		var nonRetriable *constellation.NonRetriableInitError