name: e2e test upgrade on: workflow_dispatch: inputs: attestationVariant: description: "Which attestation variant to use." type: choice options: - "aws-sev-snp" - "azure-sev-snp" - "azure-tdx" - "gcp-sev-es" - "gcp-sev-snp" default: "azure-sev-snp" required: true nodeCount: description: "Number of nodes to use in the cluster. Given in format `:`." default: "3:2" type: string fromVersion: description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'. type: string required: true gitRef: description: Ref to build upgrading CLI on, empty for HEAD. type: string default: "head" required: false toImage: description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. type: string required: false toKubernetes: description: Kubernetes version to target for the upgrade, empty for target's default version. type: string required: false toMicroservices: description: Microservice version to target for the upgrade, empty for target's default version. type: string required: false simulatedTargetVersion: description: Enter a version to build the CLI with. This can be used to simulate a patch-upgrade. type: string required: false regionZone: description: "Region or zone to create the cluster in. Leave empty for default region/zone." type: string workflow_call: inputs: attestationVariant: description: "Which attestation variant to use." type: string required: true nodeCount: description: "Number of nodes to use in the cluster. Given in format `:`." default: "3:2" type: string fromVersion: description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'. type: string required: true gitRef: description: Ref to build upgrading CLI on. type: string default: "head" required: false toImage: description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. type: string required: false toKubernetes: description: Kubernetes version to target for the upgrade, empty for target's default version. type: string required: false toMicroservices: description: Kubernetes version to target for the upgrade, empty for target's default version. type: string required: false simulatedTargetVersion: description: Enter a version to build the CLI with. This can be used to simulate a patch-upgrade. type: string required: false scheduled: description: Whether this is a scheduled run. type: boolean default: false required: false jobs: generate-input-parameters: name: Generate input parameters runs-on: ubuntu-22.04 permissions: id-token: write contents: read outputs: workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }} controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }} cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }} steps: - name: Split nodeCount id: split-nodeCount shell: bash run: | nodeCount="${{ inputs.nodeCount }}" workerNodes="${nodeCount##*:}" controlPlaneNodes="${nodeCount%%:*}" if [[ -z "${workerNodes}" ]] || [[ -z "{controlPlaneNodes}" ]]; then echo "Invalid nodeCount input: '${nodeCount}'." exit 1 fi echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT" echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT" - name: Split attestationVariant id: split-attestationVariant shell: bash run: | attestationVariant="${{ inputs.attestationVariant }}" cloudProvider="${attestationVariant%%-*}" echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT" build-target-cli: name: Build upgrade target version CLI runs-on: ubuntu-22.04 permissions: id-token: write checks: write contents: read packages: write steps: - name: Checkout if: inputs.gitRef == 'head' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Checkout ref if: inputs.gitRef != 'head' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 ref: ${{ inputs.gitRef }} - name: Setup Bazel & Nix uses: ./.github/actions/setup_bazel_nix - name: Log in to the Container registry uses: ./.github/actions/container_registry_login with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Simulate patch upgrade if: inputs.simulatedTargetVersion != '' run: | echo ${{ inputs.simulatedTargetVersion }} > version.txt - name: Build CLI uses: ./.github/actions/build_cli with: enterpriseCLI: true outputPath: "build/constellation" push: true - name: Upload CLI binary uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 with: name: constellation-upgrade-${{ inputs.attestationVariant }} path: build/constellation create-cluster: name: Create upgrade origin version cluster runs-on: ubuntu-22.04 permissions: id-token: write checks: write contents: read packages: write needs: [generate-input-parameters] outputs: kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }} e2e-name-prefix: ${{ steps.e2e_test.outputs.namePrefix }} steps: - name: Checkout if: inputs.gitRef == 'head' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Checkout ref if: inputs.gitRef != 'head' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 ref: ${{ inputs.gitRef }} - uses: ./.github/actions/setup_bazel_nix - name: Create cluster with 'fromVersion' CLI. id: e2e_test uses: ./.github/actions/e2e_test with: workerNodesCount: ${{ needs.generate-input-parameters.outputs.workerNodes }} controlNodesCount: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }} cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} attestationVariant: ${{ inputs.attestationVariant }} osImage: ${{ inputs.fromVersion }} isDebugImage: "false" cliVersion: ${{ inputs.fromVersion }} regionZone: ${{ inputs.regionZone }} gcpProject: constellation-e2e gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" test: "upgrade" azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} registry: ghcr.io githubToken: ${{ secrets.GITHUB_TOKEN }} awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }} awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }} awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }} clusterCreation: "cli" encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Remove Terraform plugin cache if: always() run: | rm -rf constellation-terraform/.terraform rm -rf constellation-iam-terraform/.terraform - name: Upload Working Directory if: always() uses: ./.github/actions/artifact_upload with: name: constellation-pre-test-${{ inputs.attestationVariant }} path: > ${{ steps.e2e_test.outputs.kubeconfig }} constellation-terraform constellation-iam-terraform constellation-conf.yaml constellation-state.yaml constellation-mastersecret.json encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Upload SA Key if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'gcp' uses: ./.github/actions/artifact_upload with: name: sa-key-${{ inputs.attestationVariant }} path: > gcpServiceAccountKey.json encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} e2e-upgrade: name: Run upgrade test runs-on: ubuntu-22.04 permissions: id-token: write checks: write contents: read packages: write needs: - generate-input-parameters - build-target-cli - create-cluster steps: - name: Checkout if: inputs.gitRef == 'head' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Checkout ref if: inputs.gitRef != 'head' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 ref: ${{ inputs.gitRef }} - name: Setup Bazel & Nix uses: ./.github/actions/setup_bazel_nix - name: Login to AWS uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead aws-region: eu-central-1 - name: Find latest nightly image id: find-image if: inputs.toImage == '' uses: ./.github/actions/versionsapi with: command: latest ref: main stream: nightly - name: Login to GCP (IAM service account) if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp' uses: ./.github/actions/login_gcp with: service_account: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" - name: Login to AWS (IAM role) if: needs.generate-input-parameters.outputs.cloudProvider == 'aws' uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM aws-region: eu-central-1 # extend token expiry to 6 hours to ensure constellation can terminate role-duration-seconds: 21600 - name: Login to Azure (IAM service principal) if: needs.generate-input-parameters.outputs.cloudProvider == 'azure' uses: ./.github/actions/login_azure with: azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} - name: Download CLI uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: name: constellation-upgrade-${{ inputs.attestationVariant }} path: build - name: Download Working Directory (Pre-test) uses: ./.github/actions/artifact_download with: name: constellation-pre-test-${{ inputs.attestationVariant }} encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Download SA Key if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp' uses: ./.github/actions/artifact_download with: name: sa-key-${{ inputs.attestationVariant }} encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Make Constellation executable and add to PATH if: always() run: | chmod +x build/constellation export PATH="$PATH:build" echo "build" >> "$GITHUB_PATH" - name: Migrate config id: constellation-config-migrate run: | ./build/constellation config migrate --debug - name: Upgrade IAM configuration id: constellation-iam-upgrade uses: ./.github/actions/constellation_iam_upgrade - name: Login to GCP (Cluster service account) if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'gcp' uses: ./.github/actions/login_gcp with: service_account: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" - name: Login to AWS (Cluster role) if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'aws' uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster aws-region: eu-central-1 # extend token expiry to 6 hours to ensure constellation can terminate role-duration-seconds: 21600 - name: Login to Azure (Cluster service principal) if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'azure' uses: ./.github/actions/login_azure with: azure_credentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} - name: Run upgrade test env: KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }} IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }} KUBERNETES: ${{ inputs.toKubernetes }} MICROSERVICES: ${{ inputs.toMicroservices }} WORKERNODES: ${{ needs.generate-input-parameters.outputs.workerNodes }} CONTROLNODES: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }} run: | echo "Image target: $IMAGE" echo "K8s target: $KUBERNETES" echo "Microservice target: $MICROSERVICES" if [[ -n ${MICROSERVICES} ]]; then MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES" fi if [[ -n ${KUBERNETES} ]]; then KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES" fi sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts' bazel run --test_timeout=14400 //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" - name: Remove Terraform plugin cache if: always() run: | rm -rf constellation-terraform/.terraform rm -rf constellation-iam-terraform/.terraform - name: Upload Working Directory if: always() uses: ./.github/actions/artifact_upload with: name: constellation-post-test-${{ inputs.attestationVariant }} path: | ${{ needs.create-cluster.outputs.kubeconfig }} constellation-terraform constellation-iam-terraform constellation-conf.yaml constellation-state.yaml constellation-mastersecret.json encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} clean-up: name: Clean up resources runs-on: ubuntu-22.04 permissions: id-token: write checks: write contents: read packages: write actions: write if: always() needs: [generate-input-parameters, create-cluster, e2e-upgrade] steps: - name: Checkout if: inputs.gitRef == 'head' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - name: Checkout ref if: inputs.gitRef != 'head' uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: fetch-depth: 0 ref: ${{ inputs.gitRef }} - name: Download CLI uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 with: name: constellation-upgrade-${{ inputs.attestationVariant }} path: build - name: Download Working Directory (Pre-test) if: always() && needs.e2e-upgrade.result != 'success' uses: ./.github/actions/artifact_download with: name: constellation-pre-test-${{ inputs.attestationVariant }} encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Download Working Directory (Post-test) if: always() && needs.e2e-upgrade.result == 'success' uses: ./.github/actions/artifact_download with: name: constellation-post-test-${{ inputs.attestationVariant }} encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Make Constellation executable and add to PATH if: always() run: | chmod +x build/constellation export PATH="$PATH:build" echo "build" >> "$GITHUB_PATH" - name: Always fetch logs if: always() env: KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }} run: | kubectl logs -n kube-system -l "app.kubernetes.io/name=constellation-operator" --tail=-1 > node-operator.logs kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs kubectl get nodeversions.update.edgeless.systems constellation-version -o yaml > constellation-version.yaml - name: Always upload logs if: always() uses: ./.github/actions/artifact_upload with: name: upgrade-logs-${{ inputs.attestationVariant }} path: > node-operator.logs node-maintenance-operator.logs constellation-version.yaml encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Prepare terraform state artifact upload if: always() shell: bash run: | mkdir -p to-zip cp -r constellation-terraform to-zip rm -f to-zip/constellation-terraform/plan.zip rm -rf to-zip/constellation-terraform/.terraform cp -r constellation-iam-terraform to-zip rm -rf to-zip/constellation-iam-terraform/.terraform - name: Always terminate cluster if: always() uses: ./.github/actions/constellation_destroy with: clusterCreation: "cli" kubeconfig: ${{ needs.create-cluster.outputs.kubeconfig }} cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }} gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com" - name: Always delete IAM configuration if: always() uses: ./.github/actions/constellation_iam_destroy with: cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }} gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com" - name: Update tfstate if: always() env: GH_TOKEN: ${{ github.token }} uses: ./.github/actions/update_tfstate with: name: terraform-state-${{ needs.create-cluster.outputs.e2e-name-prefix }} runID: ${{ github.run_id }} encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }} - name: Notify about failure if: | always() && ( needs.create-cluster.result != 'success' || needs.e2e-upgrade.result != 'success' ) && github.ref == 'refs/heads/main' && inputs.scheduled continue-on-error: true uses: ./.github/actions/notify_e2e_failure with: projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }} test: "upgrade" provider: ${{ needs.generate-input-parameters.outputs.cloudProvider }} attestationVariant: ${{ inputs.attestationVariant }}