constellation/.github/workflows/e2e-upgrade.yml

name: e2e test upgrade

on:
  workflow_dispatch:
    inputs:
      attestationVariant:
        description: "Which attestation variant to use."
        type: choice
        options:
          - "gcp-sev-es"
          - "azure-sev-snp"
          - "azure-tdx"
          - "aws-sev-snp"
        default: "azure-sev-snp"
        required: true
      nodeCount:
        description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
        default: "3:2"
        type: string
      fromVersion:
        description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
        type: string
        required: true
      gitRef:
        description: Ref to build upgrading CLI on, empty for HEAD.
        type: string
        default: "head"
        required: false
      toImage:
        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
        type: string
        required: false
      toKubernetes:
        description: Kubernetes version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      toMicroservices:
        description: Microservice version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      simulatedTargetVersion:
        description: Enter a version to build the CLI with. This can be used to simulate a patch-upgrade.
        type: string
        required: false
      regionZone:
        description: "Region or zone to create the cluster in. Leave empty for default region/zone."
        type: string
  workflow_call:
    inputs:
      attestationVariant:
        description: "Which attestation variant to use."
        type: string
        required: true
      nodeCount:
        description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
        default: "3:2"
        type: string
      fromVersion:
        description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
        type: string
        required: true
      gitRef:
        description: Ref to build upgrading CLI on.
        type: string
        default: "head"
        required: false
      toImage:
        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
        type: string
        required: false
      toKubernetes:
        description: Kubernetes version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      toMicroservices:
        description: Kubernetes version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      simulatedTargetVersion:
        description: Enter a version to build the CLI with. This can be used to simulate a patch-upgrade.
        type: string
        required: false
      scheduled:
        description: Whether this is a scheduled run.
        type: boolean
        default: false
        required: false

jobs:
  generate-input-parameters:
    name: Generate input parameters
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
      contents: read
    outputs:
      workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }}
      controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }}
      cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }}
    steps:
      - name: Split nodeCount
        id: split-nodeCount
        shell: bash
        run: |
          nodeCount="${{ inputs.nodeCount }}"
          workerNodes="${nodeCount##*:}"
          controlPlaneNodes="${nodeCount%%:*}"

          if [[ -z "${workerNodes}" ]] || [[ -z "{controlPlaneNodes}" ]]; then
            echo "Invalid nodeCount input: '${nodeCount}'."
            exit 1
          fi

          echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT"
          echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT"          

      - name: Split attestationVariant
        id: split-attestationVariant
        shell: bash
        run: |
          attestationVariant="${{ inputs.attestationVariant }}"
          cloudProvider="${attestationVariant%%-*}"

          echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT"          

  build-target-cli:
    name: Build upgrade target version CLI
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
      checks: write
      contents: read
      packages: write
    steps:
      - name: Checkout
        if: inputs.gitRef == 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

      - name: Checkout ref
        if: inputs.gitRef != 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ inputs.gitRef }}

      - name: Setup Bazel & Nix
        uses: ./.github/actions/setup_bazel_nix
        with:
          useCache: "true"
          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}

      - name: Log in to the Container registry
        uses: ./.github/actions/container_registry_login
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Simulate patch upgrade
        if: inputs.simulatedTargetVersion != ''
        run: |
          echo ${{ inputs.simulatedTargetVersion }} > version.txt          

      - name: Build CLI
        uses: ./.github/actions/build_cli
        with:
          enterpriseCLI: true
          outputPath: "build/constellation"
          push: true

      - name: Upload CLI binary
        uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
        with:
          name: constellation-upgrade-${{ inputs.attestationVariant }}
          path: build/constellation

  create-cluster:
    name: Create upgrade origin version cluster
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
      checks: write
      contents: read
      packages: write
    needs: [generate-input-parameters]
    outputs:
      kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
    steps:
      - name: Checkout
        if: inputs.gitRef == 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

      - name: Checkout ref
        if: inputs.gitRef != 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ inputs.gitRef }}

      - uses: ./.github/actions/setup_bazel_nix
        with:
          useCache: "true"
          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}

      - name: Create cluster with 'fromVersion' CLI.
        id: e2e_test
        uses: ./.github/actions/e2e_test
        with:
          workerNodesCount: ${{ needs.generate-input-parameters.outputs.workerNodes }}
          controlNodesCount: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }}
          cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
          attestationVariant: ${{ inputs.attestationVariant }}
          osImage: ${{ inputs.fromVersion }}
          isDebugImage: "false"
          cliVersion: ${{ inputs.fromVersion }}
          regionZone: ${{ inputs.regionZone }}
          gcpProject: constellation-e2e
          gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"
          gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
          test: "upgrade"
          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
          azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
          azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
          registry: ghcr.io
          githubToken: ${{ secrets.GITHUB_TOKEN }}
          awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
          awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
          awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
          clusterCreation: "cli"
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Remove Terraform plugin cache
        if: always()
        run: |
          rm -rf constellation-terraform/.terraform
          rm -rf constellation-iam-terraform/.terraform          

      - name: Upload Working Directory
        if: always()
        uses: ./.github/actions/artifact_upload
        with:
          name: constellation-pre-test-${{ inputs.attestationVariant }}
          path: >
            ${{ steps.e2e_test.outputs.kubeconfig }}
            constellation-terraform
            constellation-iam-terraform
            constellation-conf.yaml
            constellation-state.yaml
            constellation-mastersecret.json            
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Upload SA Key
        if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'gcp'
        uses: ./.github/actions/artifact_upload
        with:
          name: sa-key-${{ inputs.attestationVariant }}
          path: >
            gcpServiceAccountKey.json            
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

  e2e-upgrade:
    name: Run upgrade test
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
      checks: write
      contents: read
      packages: write
    needs:
      - generate-input-parameters
      - build-target-cli
      - create-cluster
    steps:
      - name: Checkout
        if: inputs.gitRef == 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

      - name: Checkout ref
        if: inputs.gitRef != 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ inputs.gitRef }}

      - name: Setup Bazel & Nix
        uses: ./.github/actions/setup_bazel_nix
        with:
          useCache: "true"
          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}

      - name: Login to AWS
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
        with:
          role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
          aws-region: eu-central-1

      - name: Find latest nightly image
        id: find-image
        if: inputs.toImage == ''
        uses: ./.github/actions/versionsapi
        with:
          command: latest
          ref: main
          stream: nightly

      - name: Login to GCP (IAM service account)
        if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp'
        uses: ./.github/actions/login_gcp
        with:
          service_account: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Login to AWS (IAM role)
        if: needs.generate-input-parameters.outputs.cloudProvider == 'aws'
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
        with:
          role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
          aws-region: eu-central-1
          # extend token expiry to 6 hours to ensure constellation can terminate
          role-duration-seconds: 21600

      - name: Login to Azure (IAM service principal)
        if: needs.generate-input-parameters.outputs.cloudProvider == 'azure'
        uses: ./.github/actions/login_azure
        with:
          azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}

      - name: Download CLI
        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
        with:
          name: constellation-upgrade-${{ inputs.attestationVariant }}
          path: build

      - name: Download Working Directory (Pre-test)
        uses: ./.github/actions/artifact_download
        with:
          name: constellation-pre-test-${{ inputs.attestationVariant }}
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Download SA Key
        if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp'
        uses: ./.github/actions/artifact_download
        with:
          name: sa-key-${{ inputs.attestationVariant }}
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Make Constellation executable and add to PATH
        if: always()
        run: |
          chmod +x build/constellation
          export PATH="$PATH:build"
          echo "build" >> "$GITHUB_PATH"          

      - name: Migrate config
        id: constellation-config-migrate
        run: |
          ./build/constellation config migrate --debug          

      - name: Upgrade IAM configuration
        id: constellation-iam-upgrade
        uses: ./.github/actions/constellation_iam_upgrade

      - name: Login to GCP (Cluster service account)
        if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'gcp'
        uses: ./.github/actions/login_gcp
        with:
          service_account: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Login to AWS (Cluster role)
        if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'aws'
        uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
        with:
          role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster
          aws-region: eu-central-1
          # extend token expiry to 6 hours to ensure constellation can terminate
          role-duration-seconds: 21600

      - name: Login to Azure (Cluster service principal)
        if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'azure'
        uses: ./.github/actions/login_azure
        with:
          azure_credentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}

      - name: Run upgrade test
        env:
          KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }}
          IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }}
          KUBERNETES: ${{ inputs.toKubernetes }}
          MICROSERVICES: ${{ inputs.toMicroservices }}
          WORKERNODES:  ${{ needs.generate-input-parameters.outputs.workerNodes }}
          CONTROLNODES: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }}
        run: |
          echo "Image target: $IMAGE"
          echo "K8s target: $KUBERNETES"
          echo "Microservice target: $MICROSERVICES"

          if [[ -n ${MICROSERVICES} ]]; then
            MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES"
          fi
          if [[ -n ${KUBERNETES} ]]; then
            KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES"
          fi

          bazel run //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG"          

      - name: Remove Terraform plugin cache
        if: always()
        run: |
          rm -rf constellation-terraform/.terraform
          rm -rf constellation-iam-terraform/.terraform          

      - name: Upload Working Directory
        if: always()
        uses: ./.github/actions/artifact_upload
        with:
          name: constellation-post-test-${{ inputs.attestationVariant }}
          path: |
            ${{ needs.create-cluster.outputs.kubeconfig }}
            constellation-terraform
            constellation-iam-terraform
            constellation-conf.yaml
            constellation-state.yaml
            constellation-mastersecret.json            
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

  clean-up:
    name: Clean up resources
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
      checks: write
      contents: read
      packages: write
    if: always()
    needs: [generate-input-parameters, create-cluster, e2e-upgrade]
    steps:
      - name: Checkout
        if: inputs.gitRef == 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

      - name: Checkout ref
        if: inputs.gitRef != 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ inputs.gitRef }}

      - name: Download CLI
        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
        with:
          name: constellation-upgrade-${{ inputs.attestationVariant }}
          path: build

      - name: Download Working Directory (Pre-test)
        if: always() && needs.e2e-upgrade.result != 'success'
        uses: ./.github/actions/artifact_download
        with:
          name: constellation-pre-test-${{ inputs.attestationVariant }}
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Download Working Directory (Post-test)
        if: always() && needs.e2e-upgrade.result == 'success'
        uses: ./.github/actions/artifact_download
        with:
          name: constellation-post-test-${{ inputs.attestationVariant }}
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Make Constellation executable and add to PATH
        if: always()
        run: |
          chmod +x build/constellation
          export PATH="$PATH:build"
          echo "build" >> "$GITHUB_PATH"          

      - name: Always fetch logs
        if: always()
        env:
          KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }}
        run: |
          kubectl logs -n kube-system -l "app.kubernetes.io/name=constellation-operator" --tail=-1 > node-operator.logs
          kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs
          kubectl get nodeversions.update.edgeless.systems constellation-version -o yaml > constellation-version.yaml          

      - name: Always upload logs
        if: always()
        uses: ./.github/actions/artifact_upload
        with:
          name: upgrade-logs-${{ inputs.attestationVariant }}
          path: >
            node-operator.logs
            node-maintenance-operator.logs
            constellation-version.yaml            
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Always terminate cluster
        if: always()
        uses: ./.github/actions/constellation_destroy
        with:
          clusterCreation: "cli"
          kubeconfig: ${{ needs.create-cluster.outputs.kubeconfig }}
          cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
          azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
          gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Always delete IAM configuration
        if: always()
        uses: ./.github/actions/constellation_iam_destroy
        with:
          cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
          azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
          gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Notify about failure
        if: |
          always() &&
          ( needs.create-cluster.result != 'success' || needs.e2e-upgrade.result != 'success' ) &&
          github.ref == 'refs/heads/main' &&
          inputs.scheduled          
        continue-on-error: true
        uses: ./.github/actions/notify_e2e_failure
        with:
          projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
          test: "upgrade"
          provider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
          attestationVariant: ${{ inputs.attestationVariant }}