ci: Delete e2e terraform state (#2874)

.github/actions/artifact_delete/delete_artifact.sh

@@ -7,6 +7,7 @@ function get_artifact_id {
   artifact_id="$(gh api \
     -H "Accept: application/vnd.github+json" \
     -H "X-GitHub-Api-Version: 2022-11-28" \
+    --paginate \
     "/repos/edgelesssys/constellation/actions/runs/$1/artifacts" --jq ".artifacts |= map(select(.name==\"$2\")) | .artifacts[0].id" || exit 1)"
   echo "$artifact_id"
 }

.github/actions/artifact_upload/action.yml

@@ -14,6 +14,10 @@ inputs:
   encryptionSecret:
     description: 'The secret to use for encrypting the files.'
     required: true
+  overwrite:
+    description: 'Overwrite an artifact with the same name.'
+    default: false
+    required: false
 
 runs:
   using: "composite"
@@ -69,3 +73,4 @@ runs:
         path: ${{ steps.tempdir.outputs.directory }}/archive.zip
         retention-days: ${{ inputs.retention-days }}
         if-no-files-found: ignore
+        overwrite: ${{ inputs.overwrite }}

.github/actions/e2e_cleanup_timeframe/action.yml

@@ -0,0 +1,46 @@
+name: E2E cleanup over timeframe
+description: Clean up old terraform resources of E2E tests
+
+inputs:
+  ghToken:
+    description:  'The github token that is used with the github CLI.'
+    required: true
+  encryptionSecret:
+    description: 'The secret to use for decrypting the artifacts.'
+    required: true
+  azure_credentials:
+    description: "Credentials authorized to create Constellation on Azure."
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Authenticate AWS
+      uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
+      with:
+        role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EDestroy
+        aws-region: eu-central-1
+
+    - name: Authenticate Azure
+      uses: ./.github/actions/login_azure
+      with:
+        azure_credentials: ${{ inputs.azure_credentials }}
+
+    - name: Authenticate GCP
+      uses: ./.github/actions/login_gcp
+      with:
+        service_account: "destroy-e2e@constellation-e2e.iam.gserviceaccount.com"
+
+    - name: Install unzip
+      uses: ./.github/actions/setup_bazel_nix
+      with:
+        nixTools: |
+          unzip
+    - name: Run cleanup
+      run: ./.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh
+      shell: bash
+      env:
+        GH_TOKEN: ${{ inputs.ghToken }}
+        ENCRYPTION_SECRET: ${{ inputs.encryptionSecret }}
+
+

.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh

@@ -0,0 +1,97 @@
+#!/bin/bash
+
+# get_e2e_test_ids_on_date gets all workflow IDs of workflows that contain "e2e" on a specific date.
+function get_e2e_test_ids_on_date {
+  ids="$(gh run list --created "$1" --status failure --json createdAt,workflowName,databaseId --jq '.[] | select(.workflowName | contains("e2e") and (contains("MiniConstellation") | not)) | .databaseId' -L1000 -R edgelesssys/constellation || exit 1)"
+  echo "$ids"
+}
+
+# download_tfstate_artifact downloads all artifacts matching the pattern terraform-state-* from a given workflow ID.
+function download_tfstate_artifact {
+  gh run download "$1" -p "terraform-state-*" -R edgelesssys/constellation > /dev/null
+}
+
+# delete_resources runs terraform destroy on the constellation-terraform subfolder of a given folder.
+function delete_resources {
+  if [ -d "$1/constellation-terraform" ]; then
+    cd "$1/constellation-terraform" || exit 1
+    terraform init > /dev/null || exit 1 # first, install plugins
+    terraform destroy -auto-approve || exit 1
+    cd ../../ || exit 1
+  fi
+}
+
+# delete_iam_config runs terraform destroy on the constellation-iam-terraform subfolder of a given folder.
+function delete_iam_config {
+  if [ -d "$1/constellation-iam-terraform" ]; then
+    cd "$1/constellation-iam-terraform" || exit 1
+    terraform init > /dev/null || exit 1 # first, install plugins
+    terraform destroy -auto-approve || exit 1
+    cd ../../ || exit 1
+  fi
+}
+
+# check if the password for artifact decryption was given
+if [[ -z $ENCRYPTION_SECRET ]]; then
+  echo "ENCRYPTION_SECRET is not set. Please set an environment variable with that secret."
+  exit 1
+fi
+
+artifact_pwd=$ENCRYPTION_SECRET
+
+shopt -s nullglob
+
+start_date=$(date "+%Y-%m-%d")
+end_date=$(date --date "-7 day" "+%Y-%m-%d")
+dates_to_clean=()
+
+# get all dates of the last week
+while [[ $end_date != "$start_date" ]]; do
+  dates_to_clean+=("$end_date")
+  end_date=$(date --date "$end_date +1 day" "+%Y-%m-%d")
+done
+
+echo "[*] retrieving run IDs for cleanup"
+database_ids=()
+for d in "${dates_to_clean[@]}"; do
+  echo "    retrieving run IDs from $d"
+  mapfile -td " " tmp < <(get_e2e_test_ids_on_date "$d")
+  database_ids+=("${tmp[*]}")
+done
+
+# cleanup database_ids
+mapfile -t database_ids < <(echo "${database_ids[@]}")
+mapfile -td " " database_ids < <(echo "${database_ids[@]}")
+
+echo "[*] downloading terraform state artifacts"
+for id in "${database_ids[@]}"; do
+  if [[ $id == *[^[:space:]]* ]]; then
+    echo "    downloading from workflow $id"
+    download_tfstate_artifact "$id"
+  fi
+done
+
+echo "[*] extracting artifacts"
+for directory in ./terraform-state-*; do
+  echo "    extracting $directory"
+
+  # extract and decrypt the artifact
+  unzip -d "${directory}" -P "$artifact_pwd" "$directory/archive.zip" > /dev/null || exit 1
+done
+
+# create terraform caching directory
+mkdir "$HOME/tf_plugin_cache"
+export TF_PLUGIN_CACHE_DIR="$HOME/tf_plugin_cache"
+echo "[*] created terraform cache directory $TF_PLUGIN_CACHE_DIR"
+
+echo "[*] deleting resources"
+for directory in ./terraform-state-*; do
+  echo "    deleting resources in $directory"
+  delete_resources "$directory"
+  echo "    deleting IAM configuration in $directory"
+  delete_iam_config "$directory"
+  echo "    deleting directory $directory"
+  rm -rf "$directory"
+done
+
+exit 0

.github/actions/update_tfstate/action.yml

@@ -0,0 +1,68 @@
+name: Update TFState
+description: "Update the terraform state artifact."
+
+inputs:
+  name: 
+    description: "The name of the artifact that contains the tfstate."
+    required: true
+  runID:
+    description: "The ID of your current run (github.run_id)."
+    required: true
+  encryptionSecret:
+    description: "The encryption secret for the artifacts."
+    required: true
+  skipDeletion:
+    description: "Don't try to delete the artifact before updating. You should only use this if you know that no artifact exists."
+    default: "false"
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check if tfstate should be deleted
+      if: always() && ${{ inputs.skipDeletion }} == "false"
+      shell: bash
+      run: |
+        if [ ! -d constellation-terraform ] && [ ! -d constellation-iam-terraform ]; then
+        echo "DELETE_TF_STATE=true" >> "$GITHUB_ENV"     
+        else
+        echo "DELETE_TF_STATE=false" >> "$GITHUB_ENV"
+        fi
+
+    - name: Delete tfstate artifact if necessary
+      if: always() && env.DELETE_TF_STATE == 'true' && ${{ inputs.skipDeletion }} == "false"
+      uses: ./.github/actions/artifact_delete
+      with:
+        name: ${{ inputs.name }}
+        workflowID: ${{ inputs.runID }}
+
+    - name: Prepare terraform state folders
+      if: always()
+      shell: bash
+      run: |
+        rm -rf to-zip/*
+        to_upload=""
+        if [ -d constellation-terraform ]; then
+        cp -r constellation-terraform to-zip
+        rm to-zip/constellation-terraform/plan.zip
+        rm -rf to-zip/constellation-terraform/.terraform
+        to_upload+="to-zip/constellation-terraform"
+        fi
+        if [ -d constellation-iam-terraform ]; then
+        cp -r constellation-iam-terraform to-zip
+        rm -rf to-zip/constellation-iam-terraform/.terraform
+        to_upload+=" to-zip/constellation-iam-terraform"
+        fi
+        echo "TO_UPLOAD=$to_upload" >> "$GITHUB_ENV"
+
+    - name: Update tfstate
+      if: always()
+      uses: ./.github/actions/artifact_upload
+      with:
+        name: ${{ inputs.name }}
+        path: >
+          ${{ env.TO_UPLOAD }}
+        encryptionSecret: ${{ inputs.encryptionSecret }}
+        overwrite: true
+
+

.github/workflows/e2e-cleanup-weekly.yml

@@ -0,0 +1,24 @@
+name: e2e weekly cleanup
+
+on:
+  schedule:
+    - cron: "0 0 * * 0" # At 00:00 every Sunday UTC
+  workflow_dispatch:
+    
+
+jobs:
+  cleanup:
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      id-token: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Cleanup
+        uses: ./.github/actions/e2e_cleanup_timeframe
+        with:
+          ghToken: ${{ secrets.GITHUB_TOKEN }}
+          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          azure_credentials: ${{ secrets.AZURE_E2E_DESTROY_CREDENTIALS }}

.github/workflows/e2e-test-daily.yml

@@ -56,6 +56,7 @@ jobs:
       checks: write
       contents: read
       packages: write
+      actions: write
     needs: [find-latest-image]
     steps:
       - name: Check out repository
@@ -122,6 +123,16 @@ jobs:
           azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
 
+      - name: Update tfstate
+        if: always()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/update_tfstate
+        with:
+          name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} 
+          runID: ${{ github.run_id }}
+          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+
       - name: Notify about failure
         if: |
           failure() &&

.github/workflows/e2e-test-release.yml

@@ -303,6 +303,7 @@ jobs:
       checks: write
       contents: read
       packages: write
+      actions: write
     steps:
       - name: Install the basics tools (macOS)
         if: runner.os == 'macOS'
@@ -378,6 +379,16 @@ jobs:
           azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
 
+      - name: Update tfstate
+        if: always()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/update_tfstate
+        with:
+          name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} 
+          runID: ${{ github.run_id }}
+          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+
   e2e-upgrade:
     strategy:
       fail-fast: false
@@ -392,6 +403,7 @@ jobs:
       contents: read
       checks: write
       packages: write
+      actions: write
     uses: ./.github/workflows/e2e-upgrade.yml
     with:
       fromVersion: ${{ matrix.fromVersion }}

.github/workflows/e2e-test-weekly.yml

@@ -309,6 +309,7 @@ jobs:
       checks: write
       contents: read
       packages: write
+      actions: write
     needs: [find-latest-image]
     steps:
       - name: Check out repository
@@ -378,6 +379,16 @@ jobs:
           azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
 
+      - name: Update tfstate
+        if: always()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/update_tfstate
+        with:
+          name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} 
+          runID: ${{ github.run_id }}
+          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+
       - name: Notify about failure
         if: |
           failure() &&
@@ -408,6 +419,7 @@ jobs:
       checks: write
       contents: read
       packages: write
+      actions: write
     uses: ./.github/workflows/e2e-upgrade.yml
     with:
       fromVersion: ${{ matrix.fromVersion }}
@@ -464,9 +476,9 @@ jobs:
     name: Run Windows E2E test
     permissions:
       id-token: write
-      checks: write
       contents: read
       packages: write
+      checks: write
     secrets: inherit
     uses: ./.github/workflows/e2e-windows.yml
     with:

.github/workflows/e2e-test.yml

@@ -200,6 +200,7 @@ jobs:
       checks: write
       contents: read
       packages: write
+      actions: write
     needs: [find-latest-image, generate-input-parameters]
     if: always() && !cancelled()
     steps:
@@ -278,3 +279,13 @@ jobs:
           cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
           azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
+
+      - name: Update tfstate
+        if: always()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/update_tfstate
+        with:
+          name: terraform-state-${{ steps.e2e_test.outputs.namePrefix }} 
+          runID: ${{ github.run_id }}
+          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}

.github/workflows/e2e-upgrade.yml

@@ -189,6 +189,7 @@ jobs:
     needs: [generate-input-parameters]
     outputs:
       kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+      e2e-name-prefix: ${{ steps.e2e_test.outputs.namePrefix }}
     steps:
       - name: Checkout
         if: inputs.gitRef == 'head'
@@ -441,6 +442,7 @@ jobs:
       checks: write
       contents: read
       packages: write
+      actions: write
     if: always()
     needs: [generate-input-parameters, create-cluster, e2e-upgrade]
     steps:
@@ -505,6 +507,17 @@ jobs:
             constellation-version.yaml
           encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
 
+      - name: Prepare terraform state artifact upload
+        if: always()
+        shell: bash
+        run: |
+          mkdir -p to-zip
+          cp -r constellation-terraform to-zip
+          rm to-zip/constellation-terraform/plan.zip
+          rm -rf to-zip/constellation-terraform/.terraform
+          cp -r constellation-iam-terraform to-zip
+          rm -rf to-zip/constellation-iam-terraform/.terraform
+
       - name: Always terminate cluster
         if: always()
         uses: ./.github/actions/constellation_destroy
@@ -523,6 +536,16 @@ jobs:
           azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
           gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
 
+      - name: Update tfstate
+        if: always()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/update_tfstate
+        with:
+          name: terraform-state-${{ needs.create-cluster.outputs.e2e-name-prefix }} 
+          runID: ${{ github.run_id }}
+          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+
       - name: Notify about failure
         if: |
           always() &&

.github/workflows/e2e-windows.yml

@@ -195,3 +195,26 @@ jobs:
           test: Windows E2E Test
           provider: Azure
           attestationVariant: "azure-sev-snp"
+
+  upload-tfstate:
+    name: Upload terraform state if it exists
+    runs-on: ubuntu-22.04
+    needs: e2e-test
+    if: always()
+    steps:
+      - name: Checkout
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
+
+      - name: Upload tfstate
+        if: always()
+        env:
+          GH_TOKEN: ${{ github.token }}
+        uses: ./.github/actions/update_tfstate
+        with:
+          name: terraform-state-${{ github.run_id }} 
+          runID: ${{ github.run_id }}
+          encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
+          skipDeletion: "true"
+