constellation/.github/workflows/e2e-upgrade.yml

name: e2e test upgrade

on:
  workflow_dispatch:
    inputs:
      cloudProvider:
        description: "Which cloud provider to use."
        type: choice
        options:
          - "gcp"
          - "azure"
          - "aws"
        default: "azure"
      nodeCount:
        description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
        default: "3:2"
        type: string
      fromVersion:
        description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
        type: string
        required: true
      gitRef:
        description: Ref to build upgrading CLI on, empty for HEAD.
        type: string
        default: "head"
        required: false
      toImage:
        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
        type: string
        required: false
      toKubernetes:
        description: Kubernetes version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      toMicroservices:
        description: Microservice version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      simulatedTargetVersion:
        description: Enter a version to build the CLI with. This can be used to simulate a patch-upgrade.
        type: string
        required: false
      regionZone:
        description: "Region or zone to create the cluster in. Leave empty for default region/zone."
        type: string
  workflow_call:
    inputs:
      cloudProvider:
        description: "Which cloud provider to use."
        type: string
        required: true
      nodeCount:
        description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
        default: "3:2"
        type: string
      fromVersion:
        description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
        type: string
        required: true
      gitRef:
        description: Ref to build upgrading CLI on.
        type: string
        default: "head"
        required: false
      toImage:
        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
        type: string
        required: false
      toKubernetes:
        description: Kubernetes version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      toMicroservices:
        description: Kubernetes version to target for the upgrade, empty for target's default version.
        type: string
        required: false
      simulatedTargetVersion:
        description: Enter a version to build the CLI with. This can be used to simulate a patch-upgrade.
        type: string
        required: false
      scheduled:
        description: Whether this is a scheduled run.
        type: boolean
        default: false
        required: false

jobs:
  split-nodeCount:
    name: Split nodeCount
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
      contents: read
    outputs:
      workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }}
      controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }}
    steps:
      - name: Split nodeCount
        id: split-nodeCount
        shell: bash
        run: |
          nodeCount="${{ inputs.nodeCount }}"
          workerNodes="${nodeCount##*:}"
          controlPlaneNodes="${nodeCount%%:*}"

          if [[ -z "${workerNodes}" ]] || [[ -z "{controlPlaneNodes}" ]]; then
            echo "Invalid nodeCount input: '${nodeCount}'."
            exit 1
          fi

          echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT"
          echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT"          

  e2e-upgrade:
    runs-on: ubuntu-22.04
    permissions:
      id-token: write
      checks: write
      contents: read
      packages: write
    needs: [split-nodeCount]
    steps:
      - name: Checkout
        if: inputs.gitRef == 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}

      - name: Checkout ref
        if: inputs.gitRef != 'head'
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          fetch-depth: 0
          ref: ${{ inputs.gitRef }}

      - uses: ./.github/actions/setup_bazel_nix
        with:
          useCache: "true"
          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}

      - name: Login to AWS
        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
        with:
          role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
          aws-region: eu-central-1

      - name: Find latest nightly image
        id: find-image
        if: inputs.toImage == ''
        uses: ./.github/actions/versionsapi
        with:
          command: latest
          ref: main
          stream: nightly

      - name: Simulate patch upgrade
        if: inputs.simulatedTargetVersion != ''
        run: |
          echo ${{ inputs.simulatedTargetVersion }} > version.txt          

      - name: Create cluster with 'fromVersion' CLI.
        id: e2e_test
        uses: ./.github/actions/e2e_test
        with:
          workerNodesCount: ${{ needs.split-nodeCount.outputs.workerNodes }}
          controlNodesCount: ${{ needs.split-nodeCount.outputs.controlPlaneNodes }}
          cloudProvider: ${{ inputs.cloudProvider }}
          osImage: ${{ inputs.fromVersion }}
          isDebugImage: "false"
          cliVersion: ${{ inputs.fromVersion }}
          regionZone: ${{ inputs.regionZone }}
          gcpProject: constellation-e2e
          gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"
          gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
          test: "upgrade"
          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
          azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
          azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
          registry: ghcr.io
          githubToken: ${{ secrets.GITHUB_TOKEN }}
          awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
          awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
          awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
          clusterCreation: "cli"
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Build CLI
        uses: ./.github/actions/build_cli
        with:
          enterpriseCLI: true
          outputPath: "build/constellation"
          push: true

      - name: Login to GCP (IAM service account)
        if: inputs.cloudProvider == 'gcp'
        uses: ./.github/actions/login_gcp
        with:
          service_account: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Login to AWS (IAM role)
        if: inputs.cloudProvider == 'aws'
        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
        with:
          role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
          aws-region: eu-central-1
          # extend token expiry to 6 hours to ensure constellation can terminate
          role-duration-seconds: 21600

      - name: Login to Azure (IAM service principal)
        if: inputs.cloudProvider == 'azure'
        uses: ./.github/actions/login_azure
        with:
          azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}

      - name: Migrate config
        id: constellation-config-migrate
        run: |
          ./build/constellation config migrate --debug          

      - name: Upgrade IAM configuration
        id: constellation-iam-upgrade
        uses: ./.github/actions/constellation_iam_upgrade

      - name: Login to GCP (Cluster service account)
        if: always() && inputs.cloudProvider == 'gcp'
        uses: ./.github/actions/login_gcp
        with:
          service_account: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Login to AWS (Cluster role)
        if: always() && inputs.cloudProvider == 'aws'
        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
        with:
          role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster
          aws-region: eu-central-1
          # extend token expiry to 6 hours to ensure constellation can terminate
          role-duration-seconds: 21600

      - name: Login to Azure (Cluster service principal)
        if: always() && inputs.cloudProvider == 'azure'
        uses: ./.github/actions/login_azure
        with:
          azure_credentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}

      - name: Run upgrade test
        env:
          KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }}
          IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }}
          KUBERNETES: ${{ inputs.toKubernetes }}
          MICROSERVICES: ${{ inputs.toMicroservices }}
          WORKERNODES:  ${{ needs.split-nodeCount.outputs.workerNodes }}
          CONTROLNODES: ${{ needs.split-nodeCount.outputs.controlPlaneNodes }}
        run: |
          echo "Image target: $IMAGE"
          echo "K8s target: $KUBERNETES"
          echo "Microservice target: $MICROSERVICES"

          if [[ -n ${MICROSERVICES} ]]; then
            MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES"
          fi
          if [[ -n ${KUBERNETES} ]]; then
            KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES"
          fi

          bazel run //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG"          

      - name: Always fetch logs
        if: always()
        env:
          KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }}
        run: |
          kubectl logs -n kube-system -l "app.kubernetes.io/name=constellation-operator" --tail=-1 > node-operator.logs
          kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs
          kubectl get nodeversions.update.edgeless.systems constellation-version -o yaml > constellation-version.yaml          

      - name: Always upload logs
        if: always()
        uses: ./.github/actions/artifact_upload
        with:
          name: upgrade-logs
          path: >
            node-operator.logs
            node-maintenance-operator.logs
            constellation-version.yaml            
          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

      - name: Always terminate cluster
        if: always()
        uses: ./.github/actions/constellation_destroy
        with:
          kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
          clusterCreation: "cli"
          cloudProvider: ${{ inputs.cloudProvider }}
          azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
          gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Always delete IAM configuration
        if: always()
        uses: ./.github/actions/constellation_iam_destroy
        with:
          cloudProvider: ${{ inputs.cloudProvider }}
          azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
          gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"

      - name: Notify about failure
        if: |
          failure() &&
          github.ref == 'refs/heads/main' &&
          inputs.scheduled          
        continue-on-error: true
        uses: ./.github/actions/notify_e2e_failure
        with:
          projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
          test: "upgrade"
          provider: ${{ inputs.cloudProvider }}