ci: use 7zip for creating archives (#3068)

* Use 7zip for creating and processing encrypted archives
* Switch to .7z file extension
* Fix shell check issues
* Fix tfstate update logic

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

.github/actions/artifact_download/action.yml

@@ -16,11 +16,11 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install unzip
+    - name: Install 7zip
       uses: ./.github/actions/setup_bazel_nix
       with:
         nixTools: |
-          unzip
+          _7zz
 
     - name: Create temporary directory
       id: tempdir
@@ -37,4 +37,4 @@ runs:
       shell: bash
       run: |
         mkdir -p ${{ inputs.path }}
-        unzip -P '${{ inputs.encryptionSecret }}' -qq -d ${{ inputs.path }} ${{ steps.tempdir.outputs.directory }}/archive.zip
+        7zz x -p'${{ inputs.encryptionSecret }}' -t7z -o"${{ inputs.path }}" ${{ steps.tempdir.outputs.directory }}/archive.7z

.github/actions/artifact_upload/action.yml

@@ -22,13 +22,51 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Install zip
+    - name: Install 7zip
       uses: ./.github/actions/setup_bazel_nix
       with:
         nixTools: |
-          zip
+          _7zz
 
     - name: Create temporary directory
       id: tempdir
       shell: bash
       run: echo "directory=$(mktemp -d)" >> "$GITHUB_OUTPUT"
+
+    - name: Create archive
+      shell: bash
+      run: |
+        shopt -s extglob
+        paths="${{ inputs.path }}"
+        paths=${paths%$'\n'} # Remove trailing newline
+        # Check if any file matches the given pattern(s).
+        something_exists=false
+        for pattern in ${paths}
+        do
+          if compgen -G "${pattern}" > /dev/null; then
+            something_exists=true
+          fi
+        done
+        # Create an archive if files exist.
+        # Don't create an archive file if no files are found
+        # and warn.
+        if ! ${something_exists}
+        then
+          echo "::warning:: No files/directories found with the provided path(s): ${paths}. No artifact will be uploaded."
+          exit 0
+        fi
+        for target in ${paths}
+        do
+          pushd "$(dirname "${target}")" || exit 1
+          7zz a -p'${{ inputs.encryptionSecret }}' -t7z -ms=on -mhe=on "${{ steps.tempdir.outputs.directory }}/archive.7z" "$(basename "${target}")"
+          popd || exit 1
+        done
+
+    - name: Upload archive as artifact
+      uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+      with:
+        name: ${{ inputs.name }}
+        path: ${{ steps.tempdir.outputs.directory }}/archive.7z
+        retention-days: ${{ inputs.retention-days }}
+        if-no-files-found: ignore
+        overwrite: ${{ inputs.overwrite }}

.github/actions/e2e_cleanup_timeframe/action.yml

@@ -31,11 +31,11 @@ runs:
       with:
         service_account: "destroy-e2e@constellation-e2e.iam.gserviceaccount.com"
 
-    - name: Install unzip
+    - name: Install 7zip
       uses: ./.github/actions/setup_bazel_nix
       with:
         nixTools: |
-          unzip
+          _7zz
     - name: Run cleanup
       run: ./.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh
       shell: bash

.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh

@@ -3,7 +3,7 @@
 # get_e2e_test_ids_on_date gets all workflow IDs of workflows that contain "e2e" on a specific date.
 function get_e2e_test_ids_on_date {
   ids="$(gh run list --created "$1" --status failure --json createdAt,workflowName,databaseId --jq '.[] | select(.workflowName | contains("e2e") and (contains("MiniConstellation") | not)) | .databaseId' -L1000 -R edgelesssys/constellation || exit 1)"
-  echo "$ids"
+  echo "${ids}"
 }
 
 # download_tfstate_artifact downloads all artifacts matching the pattern terraform-state-* from a given workflow ID.
@@ -13,7 +13,7 @@ function download_tfstate_artifact {
 
 # delete_resources runs terraform destroy on the constellation-terraform subfolder of a given folder.
 function delete_resources {
-  if [ -d "$1/constellation-terraform" ]; then
+  if [[ -d "$1/constellation-terraform" ]]; then
     cd "$1/constellation-terraform" || exit 1
     terraform init > /dev/null || exit 1 # first, install plugins
     terraform destroy -auto-approve || exit 1
@@ -23,7 +23,7 @@ function delete_resources {
 
 # delete_iam_config runs terraform destroy on the constellation-iam-terraform subfolder of a given folder.
 function delete_iam_config {
-  if [ -d "$1/constellation-iam-terraform" ]; then
+  if [[ -d "$1/constellation-iam-terraform" ]]; then
     cd "$1/constellation-iam-terraform" || exit 1
     terraform init > /dev/null || exit 1 # first, install plugins
     terraform destroy -auto-approve || exit 1
@@ -32,12 +32,12 @@ function delete_iam_config {
 }
 
 # check if the password for artifact decryption was given
-if [[ -z $ENCRYPTION_SECRET ]]; then
+if [[ -z ${ENCRYPTION_SECRET} ]]; then
   echo "ENCRYPTION_SECRET is not set. Please set an environment variable with that secret."
   exit 1
 fi
 
-artifact_pwd=$ENCRYPTION_SECRET
+artifact_pwd=${ENCRYPTION_SECRET}
 
 shopt -s nullglob
 
@@ -46,9 +46,9 @@ end_date=$(date --date "-7 day" "+%Y-%m-%d")
 dates_to_clean=()
 
 # get all dates of the last week
-while [[ $end_date != "$start_date" ]]; do
+while [[ ${end_date} != "${start_date}" ]]; do
-  dates_to_clean+=("$end_date")
+  dates_to_clean+=("${end_date}")
-  end_date=$(date --date "$end_date +1 day" "+%Y-%m-%d")
+  end_date=$(date --date "${end_date} +1 day" "+%Y-%m-%d")
 done
 
 echo "[*] retrieving run IDs for cleanup"
@@ -65,33 +65,33 @@ mapfile -td " " database_ids < <(echo "${database_ids[@]}")
 
 echo "[*] downloading terraform state artifacts"
 for id in "${database_ids[@]}"; do
-  if [[ $id == *[^[:space:]]* ]]; then
+  if [[ ${id} == *[^[:space:]]* ]]; then
-    echo "    downloading from workflow $id"
+    echo "    downloading from workflow ${id}"
-    download_tfstate_artifact "$id"
+    download_tfstate_artifact "${id}"
   fi
 done
 
 echo "[*] extracting artifacts"
 for directory in ./terraform-state-*; do
-  echo "    extracting $directory"
+  echo "    extracting ${directory}"
 
   # extract and decrypt the artifact
-  unzip -d "${directory}" -P "$artifact_pwd" "$directory/archive.zip" > /dev/null || exit 1
+  7zz x -t7z -p"${artifact_pwd}" -o"${directory}" "${directory}/archive.7z" > /dev/null || exit 1
 done
 
 # create terraform caching directory
-mkdir "$HOME/tf_plugin_cache"
+mkdir "${HOME}/tf_plugin_cache"
-export TF_PLUGIN_CACHE_DIR="$HOME/tf_plugin_cache"
+export TF_PLUGIN_CACHE_DIR="${HOME}/tf_plugin_cache"
-echo "[*] created terraform cache directory $TF_PLUGIN_CACHE_DIR"
+echo "[*] created terraform cache directory ${TF_PLUGIN_CACHE_DIR}"
 
 echo "[*] deleting resources"
 for directory in ./terraform-state-*; do
-  echo "    deleting resources in $directory"
+  echo "    deleting resources in ${directory}"
-  delete_resources "$directory"
+  delete_resources "${directory}"
-  echo "    deleting IAM configuration in $directory"
+  echo "    deleting IAM configuration in ${directory}"
-  delete_iam_config "$directory"
+  delete_iam_config "${directory}"
-  echo "    deleting directory $directory"
+  echo "    deleting directory ${directory}"
-  rm -rf "$directory"
+  rm -rf "${directory}"
 done
 
 exit 0

.github/actions/update_tfstate/action.yml

@@ -1,5 +1,5 @@
 name: Update TFState
-description: "Update the terraform state artifact."
+description: "Update the terraform state artifact. We use this to either delete an artifact if the e2e test was cleaned up successfully or to update the artifact with the latest terraform state."
 
 inputs:
   name:
@@ -11,33 +11,29 @@ inputs:
   encryptionSecret:
     description: "The encryption secret for the artifacts."
     required: true
-  skipDeletion:
-    description: "Don't try to delete the artifact before updating. You should only use this if you know that no artifact exists."
-    default: "false"
-    required: false
 
 runs:
   using: "composite"
   steps:
-    - name: Check if tfstate should be deleted
+    - name: Check if uploaded tfstate can be deleted
-      if: always() && inputs.skipDeletion == 'false'
+      if: always()
       shell: bash
       run: |
-        if [[ -d constellation-terraform ]] || [[ -d constellation-iam-terraform ]]; then
+        if [[ ! -d constellation-terraform ]] && [[ ! -d constellation-iam-terraform ]]; then
           echo "DELETE_TF_STATE=true" >> "$GITHUB_ENV"
         else
           echo "DELETE_TF_STATE=false" >> "$GITHUB_ENV"
         fi
 
     - name: Delete tfstate artifact if necessary
-      if: always() && env.DELETE_TF_STATE == 'true' && inputs.skipDeletion == 'false'
+      if: always() && env.DELETE_TF_STATE == 'true'
       uses: ./.github/actions/artifact_delete
       with:
         name: ${{ inputs.name }}
         workflowID: ${{ inputs.runID }}
 
-    - name: Prepare terraform state folders
+    - name: Prepare left over terraform state folders
-      if: always()
+      if: always() && env.DELETE_TF_STATE == 'false'
       shell: bash
       run: |
         rm -rf to-zip/*