constellation/.github/workflows/e2e-test-provider-example.yml

name: e2e test Terraform provider example

on:
  workflow_dispatch:
    inputs:
      ref:
        type: string
        description: "Git ref to checkout"
      regionZone:
        description: "Region or zone to create the cluster in. Leave empty for default region/zone."
        type: string
      image:
        description: "OS Image version used in the cluster's VMs. If not set, the latest nightly image from main is used."
        type: string
      providerVersion:
        description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source."
        type: string
      toImage:
        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
        type: string
        required: false
      toKubernetes:
        description: Kubernetes version to target for the upgrade, empty for no upgrade.
        type: string
        required: false
      attestationVariant:
        description: "Attestation variant to use."
        type: choice
        options:
          - "aws-sev-snp"
          - "azure-sev-snp"
          - "azure-tdx"
          - "gcp-sev-es"
        default: "azure-sev-snp"
        required: true
  workflow_call:
    inputs:
      ref:
        type: string
        description: "Git ref to checkout"
      regionZone:
        description: "Which zone to use."
        type: string
      image:
        description: "OS Image version used in the cluster's VMs, as specified in the Constellation config. If not set, the latest nightly image from main is used."
        type: string
      providerVersion:
        description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source."
        type: string
      toImage:
        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
        type: string
        required: false
      toKubernetes:
        description: Kubernetes version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      attestationVariant:
        description: "Attestation variant to use."
        type: string
        required: true

jobs:
  provider-example-test:
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
      contents: read
      packages: write
    steps:
      - name: Checkout
        id: checkout
        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
        with:
          ref: ${{ inputs.ref || github.head_ref }}

      - name: Get Latest Image
        id: find-latest-image
        uses: ./.github/actions/find_latest_image
        with:
          git-ref: ${{ inputs.ref }}
          imageVersion: ${{ inputs.image }}
          ref: main
          stream: nightly

      - name: Determine cloudprovider from attestation variant
        id: determine
        shell: bash
        run: |
          attestationVariant="${{ inputs.attestationVariant }}"
          cloudProvider="${attestationVariant%%-*}"

          echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT"          

      - name: Log in to the Container registry
        uses: ./.github/actions/container_registry_login
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Download CLI # needed to determine K8s version for release versions
        if: inputs.providerVersion != ''
        shell: bash
        run: |
          curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/constellation-linux-amd64
          chmod u+x constellation
          ./constellation version
          mkdir -p ${{ github.workspace }}/release
          cp ./constellation ${{ github.workspace }}/release          

      - name: Setup bazel
        uses: ./.github/actions/setup_bazel_nix
        with:
          useCache: "true"
          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
          nixTools: terraform

      - name: Create prefix
        id: create-prefix
        shell: bash
        run: |
          uuid=$(uuidgen | tr "[:upper:]" "[:lower:]")
          uuid="${uuid%%-*}"
          uuid="${uuid: -3}" # Final resource name must be no longer than 10 characters on AWS
          echo "uuid=${uuid}" | tee -a "${GITHUB_OUTPUT}"
          echo "prefix=e2e-${uuid}" | tee -a "${GITHUB_OUTPUT}"          

      - name: Build Constellation provider and CLI # CLI is needed for the upgrade assert and container push is needed for the microservice upgrade
        working-directory: ${{ github.workspace }}
        id: build
        shell: bash
        run: |
          mkdir -p ${{ github.workspace }}/build
          cd ${{ github.workspace }}/build
          bazel run //:devbuild --cli_edition=enterprise

          bazel build //bazel/settings:tag
          repository_root=$(git rev-parse --show-toplevel)
          out_rel=$(bazel cquery --output=files //bazel/settings:tag)
          build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")
          echo "build_version=${build_version}" | tee -a "$GITHUB_OUTPUT"          

      - name: Remove local Terraform registry # otherwise the local registry would be used instead of the public registry
        if: inputs.providerVersion != ''
        shell: bash
        run: |
          bazel build //bazel/settings:tag
          repository_root=$(git rev-parse --show-toplevel)
          out_rel=$(bazel cquery --output=files //bazel/settings:tag)
          build_version=$(cat "$(realpath "${repository_root}/${out_rel}")")

          terraform_provider_dir="${HOME}/.terraform.d/plugins/registry.terraform.io/edgelesssys/constellation/${build_version#v}/linux_amd64/"
          rm -rf "${terraform_provider_dir}"          

      - name: Login to AWS (IAM + Cluster role)
        if: steps.determine.outputs.cloudProvider == 'aws'
        uses: aws-actions/configure-aws-credentials@5fd3084fc36e372ff1fff382a39b10d03659f355 # v2.2.0
        with:
          role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ETerraform
          aws-region: eu-central-1
          # extend token expiry to 6 hours to ensure constellation can terminate
          role-duration-seconds: 21600

      - name: Login to Azure (IAM + Cluster service principal)
        if: steps.determine.outputs.cloudProvider == 'azure'
        uses: ./.github/actions/login_azure
        with:
          azure_credentials: ${{ secrets.AZURE_E2E_TF_CREDENTIALS }}

      - name: Login to GCP (IAM + Cluster service account)
        if: steps.determine.outputs.cloudProvider == 'gcp'
        uses: ./.github/actions/login_gcp
        with:
          service_account: "terraform-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Set Kubernetes version
        id: kubernetes
        run: |
          set -e

          # take the middle (2nd) supported Kubernetes version (default)
          if [[ "${{ inputs.providerVersion }}" != "" ]]; then
            cli_output=$(${{ github.workspace }}/release/constellation config kubernetes-versions)
          else
            cli_output=$(${{ github.workspace }}/build/constellation config kubernetes-versions)
          fi
          echo "version=$(echo "${cli_output}" | awk 'NR==3{print $1}')" | tee -a "${GITHUB_OUTPUT}"          

      - name: Common CSP Terraform overrides
        working-directory: ${{ github.workspace }}
        shell: bash
        run: |
          mkdir -p ${{ github.workspace }}/cluster
          cd ${{ github.workspace }}/cluster
          if [[ "${{ inputs.providerVersion }}" == "" ]]; then
            prefixed_version=${{ steps.build.outputs.build_version }}
          else
            prefixed_version="${{ inputs.providerVersion }}"
          fi
          version=${prefixed_version#v} # remove v prefix

          if [[ "${{ inputs.providerVersion }}" == "" ]]; then
            iam_src="${{ github.workspace }}/terraform/infrastructure/iam/${{ steps.determine.outputs.cloudProvider }}"
            infra_src="${{ github.workspace }}/terraform/infrastructure/${{ steps.determine.outputs.cloudProvider }}"
          else
            iam_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/iam/${{ steps.determine.outputs.cloudProvider }}"
            infra_src="https://github.com/edgelesssys/constellation/releases/download/${{ inputs.providerVersion }}/terraform-module.zip//terraform-module/${{ steps.determine.outputs.cloudProvider }}"
          fi

          # by default use latest nightly image for devbuilds and release image otherwise
          if [[ "${{ inputs.providerVersion }}" == "" ]]; then
            if [[ "${{ inputs.image }}" == "" ]]; then
              image_version="${{ steps.find-latest-image.outputs.image }}"
            else
              image_version="${{ inputs.image }}"
            fi
          else
            if [[ "${{ inputs.image }}" == "" ]]; then
              image_version="${prefixed_version}"
            else
              image_version="${{ inputs.image }}"
            fi
          fi

          kubernetes_version="${{ steps.kubernetes.outputs.version }}"

          cat > _override.tf <<EOF
          terraform {
            required_providers {
              constellation = {
                source  = "edgelesssys/constellation"
                version = "${version}"
              }
            }
          }

          locals {
            control_plane_count = 1
            worker_count        = 1
          }

          locals {
            name = "${{ steps.create-prefix.outputs.prefix }}"
            image_version = "${image_version}"
            microservice_version = "${prefixed_version}"
            kubernetes_version = "${kubernetes_version}"
            attestation_variant = "${{ inputs.attestationVariant }}"
          }

          module "${{ steps.determine.outputs.cloudProvider }}_iam" {
            source = "${iam_src}"
          }

          module "${{ steps.determine.outputs.cloudProvider }}_infrastructure" {
            source = "${infra_src}"
          }
          EOF
          cat _override.tf          

      - name: Create GCP Terraform overrides
        if: steps.determine.outputs.cloudProvider == 'gcp'
        working-directory: ${{ github.workspace }}/cluster
        shell: bash
        run: |
          region=$(echo ${{ inputs.regionZone || 'europe-west3-b' }} | rev | cut -c 3- | rev)

          cat >> _override.tf <<EOF
          locals {
            project_id         = "constellation-e2e"
            region = "${region}"
            zone = "${{ inputs.regionZone || 'europe-west3-b' }}"
          }
          EOF
          cat _override.tf          

      - name: Create AWS Terraform overrides
        if: steps.determine.outputs.cloudProvider == 'aws'
        working-directory: ${{ github.workspace }}/cluster
        shell: bash
        run: |
          region=$(echo ${{ inputs.regionZone || 'us-east-2c' }} | rev | cut -c 2- | rev)

          cat >> _override.tf <<EOF
          locals {
            region = "${region}"
            zone = "${{ inputs.regionZone || 'us-east-2c' }}"
          }
          EOF
          cat _override.tf          

      - name: Create Azure TDX Terraform overrides
        if: inputs.attestationVariant == 'azure-tdx'
        working-directory: ${{ github.workspace }}/cluster
        shell: bash
        run: |
          cat >> _override.tf <<EOF
          locals {
            instance_type = "Standard_DC4es_v5"
          }
          EOF
          cat _override.tf          

      - name: Copy example Terraform file
        working-directory: ${{ github.workspace }}
        shell: bash
        run: |
          cp ${{ github.workspace }}/terraform-provider-constellation/examples/full/${{ steps.determine.outputs.cloudProvider }}/main.tf ${{ github.workspace }}/cluster/main.tf          

      - name: Apply Terraform Cluster
        id: apply_terraform
        working-directory: ${{ github.workspace }}/cluster
        shell: bash
        run: |
          sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts'
          terraform init
          if [[ "${{ inputs.attestationVariant }}" == "azure-sev-snp" ]]; then
            terraform apply -target module.azure_iam -auto-approve
            terraform apply -target module.azure_infrastructure -auto-approve
            ${{ github.workspace }}/build/constellation maa-patch "$(terraform output -raw maa_url)"
            terraform apply -target constellation_cluster.azure_example -auto-approve
          else
            terraform apply -auto-approve
          fi          

      - name: Cleanup Terraform Cluster on failure
        # cleanup here already on failure, because the subsequent TF overrides might make the TF config invalid and thus the destroy would fail later
        # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
        if: failure() && steps.apply_terraform.outcome != 'skipped'
        working-directory: ${{ github.workspace }}/cluster
        shell: bash
        run: |
          terraform init
          terraform destroy -auto-approve          

      - name: Add Provider to local Terraform registry # needed if release version was used before
        if: inputs.providerVersion != ''
        working-directory: ${{ github.workspace }}/build
        shell: bash
        run: |
          bazel run //:devbuild --cli_edition=enterprise          

      - name: Update cluster configuration # for duplicate variable declaration, the last one is used
        working-directory: ${{ github.workspace }}/cluster
        shell: bash
        run: |
          cat >> _override.tf <<EOF
          locals {
            image_version = "${{ inputs.toImage || steps.find-latest-image.outputs.image }}"
          }
          EOF

          if [[ "${{ inputs.toKubernetes }}" != "" ]]; then
          cat >> _override.tf <<EOF
          resource "constellation_cluster" "${{ steps.determine.outputs.cloudProvider }}_example" {
            kubernetes_version = "${{ inputs.toKubernetes }}"
          }
          EOF
          fi

          prefixed_version=${{ steps.build.outputs.build_version }}
          version=${prefixed_version#v} # remove v prefix

          # needs to be explicitly set to upgrade
          cat >> _override.tf <<EOF
          resource "constellation_cluster" "${{ steps.determine.outputs.cloudProvider }}_example" {
            constellation_microservice_version = "${prefixed_version}"
          }
          EOF

          cat >> _override.tf <<EOF
          terraform {
            required_providers {
              constellation = {
                source  = "edgelesssys/constellation"
                version = "${version}"
              }
            }
          }
          EOF
          cat _override.tf          

      - name: Upgrade Terraform Cluster
        working-directory: ${{ github.workspace }}/cluster
        shell: bash
        run: |
          terraform init --upgrade
          terraform apply -auto-approve          

      - name: Assert upgrade successful
        working-directory: ${{ github.workspace }}/cluster
        env:
          IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-latest-image.outputs.image }}
          KUBERNETES: ${{ inputs.toKubernetes }}
          MICROSERVICES: ${{ steps.build.outputs.build_version }}
          WORKERNODES: 1
          CONTROLNODES: 1
        run: |
          terraform output -raw kubeconfig > constellation-admin.conf

          if [[ -n "${MICROSERVICES}" ]]; then
            MICROSERVICES_FLAG="--target-microservices=${MICROSERVICES}"
          fi
          if [[ -n "${KUBERNETES}" ]]; then
            KUBERNETES_FLAG="--target-kubernetes=${KUBERNETES}"
          fi
          if [[ -n "${IMAGE}" ]]; then
            IMAGE_FLAG="--target-image=${IMAGE}"
          fi

          # cfg must be in same dir as KUBECONFIG
          ${{ github.workspace }}/build/constellation config generate "${{ steps.determine.outputs.cloudProvider }}" --attestation ${{ inputs.attestationVariant}}
          # make cfg valid with fake data
          # IMPORTANT: zone needs to be correct because it is used to resolve the CSP image ref
          if [[ "${{ steps.determine.outputs.cloudProvider }}" == "azure" ]]; then
            location="${{ inputs.regionZone || 'northeurope' }}"
            yq e ".provider.azure.location = \"${location}\"" -i constellation-conf.yaml

            yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml
            yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml
            yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml
            yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml
          fi
          if [[ "${{ steps.determine.outputs.cloudProvider }}" == "gcp" ]]; then
            zone="${{ inputs.regionZone || 'europe-west3-b' }}"
            region=$(echo "${zone}" | rev | cut -c 2- | rev)
            yq e ".provider.gcp.region = \"${region}\"" -i constellation-conf.yaml
            yq e ".provider.gcp.zone = \"${zone}\"" -i constellation-conf.yaml

            yq e '.provider.gcp.project = "demo-gcp-project"' -i constellation-conf.yaml
            yq e '.nodeGroups.control_plane_default.zone = "europe-west3-b"' -i constellation-conf.yaml
            # Set the zone for worker_default node group to a fictional value
            yq e '.nodeGroups.worker_default.zone = "europe-west3-b"' -i constellation-conf.yaml
            yq e '.provider.gcp.serviceAccountKeyPath = "/path/to/your/service-account-key.json"' -i constellation-conf.yaml
          fi
          if [[ "${{ steps.determine.outputs.cloudProvider }}" == "aws" ]]; then
            zone=${{ inputs.regionZone || 'us-east-2c' }}
            region=$(echo "${zone}" | rev | cut -c 2- | rev)
            yq e ".provider.aws.region = \"${region}\"" -i constellation-conf.yaml
            yq e ".provider.aws.zone = \"${zone}\"" -i constellation-conf.yaml

            yq e '.provider.aws.iamProfileControlPlane = "demoControlPlaneIAMProfile"' -i constellation-conf.yaml
            yq e '.provider.aws.iamProfileWorkerNodes = "demoWorkerNodesIAMProfile"' -i constellation-conf.yaml
            yq e '.nodeGroups.control_plane_default.zone = "eu-central-1a"' -i constellation-conf.yaml
            yq e '.nodeGroups.worker_default.zone = "eu-central-1a"' -i constellation-conf.yaml
          fi
          KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --cli "${{ github.workspace }}/build/constellation" "$IMAGE_FLAG" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG"          

      - name: Destroy Terraform Cluster
        # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context)
        if: always() && steps.apply_terraform.outcome != 'skipped'
        working-directory: ${{ github.workspace }}/cluster
        shell: bash
        run: |
          terraform init
          terraform destroy -auto-approve          

      - name: Notify about failure
        if: |
          failure() &&
          github.ref == 'refs/heads/main' &&
          github.event_name == 'schedule'          
        continue-on-error: true
        uses: ./.github/actions/notify_e2e_failure
        with:
          projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
          test: "terraform-provider-example"
          refStream: ${{ inputs.ref}}
          provider: ${{ steps.determine.outputs.cloudProvider }}
          kubernetesVersion: ${{ steps.kubernetes.outputs.version }}
          clusterCreation: "terraform"
          attestationVariant: ${{ inputs.attestationVariant }}