ci: run e2e cleanup daily (#3819)

* ci: run e2e cleanup job once daily
* ci: dont abort cleanup on first error
* ci: clean up leftover resources from release workflows

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

.github/actions/e2e_cleanup_timeframe/e2e-cleanup.sh

@@ -2,7 +2,7 @@
 
 # get_e2e_test_ids_on_date gets all workflow IDs of workflows that contain "e2e" on a specific date.
 function get_e2e_test_ids_on_date {
-  ids="$(gh run list --created "$1" --status failure --json createdAt,workflowName,databaseId --jq '.[] | select(.workflowName | contains("e2e") and (contains("MiniConstellation") | not)) | .databaseId' -L1000 -R edgelesssys/constellation || exit 1)"
+  ids="$(gh run list --created "$1" --status failure --json createdAt,workflowName,databaseId --jq '.[] | select(.workflowName | (contains("e2e") or contains("Release")) and (contains("MiniConstellation") | not)) | .databaseId' -L1000 -R edgelesssys/constellation || exit 1)"
   echo "${ids}"
 }
 
@@ -11,24 +11,15 @@ function download_tfstate_artifact {
   gh run download "$1" -p "terraform-state-*" -R edgelesssys/constellation > /dev/null
 }
 
-# delete_resources runs terraform destroy on the constellation-terraform subfolder of a given folder.
-function delete_resources {
-  if [[ -d "$1/constellation-terraform" ]]; then
-    cd "$1/constellation-terraform" || exit 1
-    terraform init > /dev/null || exit 1 # first, install plugins
-    terraform destroy -auto-approve || exit 1
-    cd ../../ || exit 1
-  fi
-}
-
-# delete_iam_config runs terraform destroy on the constellation-iam-terraform subfolder of a given folder.
-function delete_iam_config {
-  if [[ -d "$1/constellation-iam-terraform" ]]; then
-    cd "$1/constellation-iam-terraform" || exit 1
-    terraform init > /dev/null || exit 1 # first, install plugins
-    terraform destroy -auto-approve || exit 1
-    cd ../../ || exit 1
+# delete_terraform_resources runs terraform destroy on the given folder.
+function delete_terraform_resources {
+  delete_err=0
+  if pushd "${1}/${2}"; then
+    terraform init > /dev/null || delete_err=1 # first, install plugins
+    terraform destroy -auto-approve || delete_err=1
+    popd || exit 1
   fi
+  return "${delete_err}"
 }
 
 # check if the password for artifact decryption was given
@@ -42,7 +33,7 @@ artifact_pwd=${ENCRYPTION_SECRET}
 shopt -s nullglob
 
 start_date=$(date "+%Y-%m-%d")
-end_date=$(date --date "-7 day" "+%Y-%m-%d")
+end_date=$(date --date "-4 day" "+%Y-%m-%d")
 dates_to_clean=()
 
 # get all dates of the last week
@@ -85,13 +76,19 @@ export TF_PLUGIN_CACHE_DIR="${HOME}/tf_plugin_cache"
 echo "[*] created terraform cache directory ${TF_PLUGIN_CACHE_DIR}"
 
 echo "[*] deleting resources"
+error_occurred=0
 for directory in ./terraform-state-*; do
   echo "    deleting resources in ${directory}"
-  delete_resources "${directory}"
+  delete_terraform_resources "${directory}" "constellation-terraform" || echo "[!] deleting resources failed" && error_occurred=1
   echo "    deleting IAM configuration in ${directory}"
-  delete_iam_config "${directory}"
+  delete_terraform_resources "${directory}" "constellation-iam-terraform" || echo "[!] deleting IAM resources failed" && error_occurred=1
   echo "    deleting directory ${directory}"
   rm -rf "${directory}"
 done
 
+if [[ ${error_occurred} -ne 0 ]]; then
+  echo "[!] Errors occurred during resource deletion."
+  exit 1
+fi
+
 exit 0

.github/workflows/e2e-cleanup.yml

@@ -1,8 +1,8 @@
-name: e2e weekly cleanup
+name: e2e cleanup
 
 on:
   schedule:
-    - cron: "0 0 * * 0" # At 00:00 every Sunday UTC
+    - cron: "0 0 * * *" # At 00:00 every day
   workflow_dispatch: