e2e: fix deletion of persisten volumes (#476)

Co-authored-by: Christoph Meyer <cme@edgeless.systems>

.github/actions/constellation_destroy/action.yml

@@ -1,29 +1,43 @@
 name: Constellation destroy
 description: "Destroy a running Constellation cluster."
 
+inputs:
+  kubeconfig:
+    description: "The kubeconfig for the cluster."
+    required: true
+
 runs:
   using: "composite"
   steps:
-    - name: Delete PVs
+    - name: Delete persistent volumes
+      if: inputs.kubeconfig != ''
+      continue-on-error: true
       shell: bash
+      env:
+        KUBECONFIG: ${{ inputs.kubeconfig }}
+        PV_DELETION_TIMEOUT: "120" # 2 minutes timeout for pv deletion
       run: |
         ELAPSED=0
-        kubectl delete pvc --all --wait=true || true
         echo "::group::Wait for PV deletion"
+
+        # Scrap namespaces that contain PVCs
+        for namespace in `kubectl get namespace --no-headers=true -o custom-columns=":metadata.name"`; do
+          if [ `kubectl get pvc -n $namespace --no-headers=true -o custom-columns=":metadata.name" | wc -l` -gt 0 ]; then
+            kubectl delete namespace $namespace --wait
+          fi
+        done
+
         until [ "$(kubectl get pv -o json | jq '.items | length')" == "0" ] || [ $ELAPSED -gt $PV_DELETION_TIMEOUT ];
         do
-        echo $(kubectl get pv -o json | jq '.items | length') PV remaining..
+          echo $(kubectl get pv -o json | jq '.items | length') PV remaining..
-        sleep 1
+          sleep 1
-        ELAPSED=$((ELAPSED+1))
+          ELAPSED=$((ELAPSED+1))
         done
         if [ $ELAPSED -gt $PV_DELETION_TIMEOUT ]; then
-        echo "Timed out waiting for PV deletion.."
+          echo "Timed out waiting for PV deletion.."
-        exit 1
+          exit 1
         fi
         echo "::endgroup::"
-      env:
-        KUBECONFIG: ${{ github.workspace }}/constellation-admin.conf
-        PV_DELETION_TIMEOUT: "120" # 2 minutes timeout for pv deletion
 
     - name: Constellation terminate
       run: constellation terminate --yes

.github/actions/e2e_test/action.yml

@@ -61,6 +61,11 @@ inputs:
     description: "The sonobuoy test suite to run."
     required: false
 
+outputs:
+  kubeconfig:
+    description: "The kubeconfig for the cluster."
+    value: ${{ steps.constellation-create.outputs.kubeconfig }}
+
 runs:
   using: "composite"
   steps:

.github/actions/k-bench/action.yml

@@ -63,27 +63,6 @@ runs:
       env:
         KUBECONFIG: ${{ inputs.kubeconfig }}
 
-    - name: Delete namespace and PVCs
-      shell: bash
-      run: |
-        ELAPSED=0
-        kubectl delete namespace kbench-pod-namespace --wait=true || true
-        echo "::group::Wait for PV deletion"
-        until [ "$(kubectl get pv -o json | jq '.items | length')" == "0" ] || [ $ELAPSED -gt $PV_DELETION_TIMEOUT ];
-        do
-          echo $(kubectl get pv -o json | jq '.items | length') PV remaining..
-          sleep 1
-          ELAPSED=$((ELAPSED+1))
-        done
-        if [ $ELAPSED -gt $PV_DELETION_TIMEOUT ]; then
-          echo "Timed out waiting for PV deletion.."
-          exit 1
-        fi
-        echo "::endgroup::"
-      env:
-        KUBECONFIG: ${{ inputs.kubeconfig }}
-        PV_DELETION_TIMEOUT: "120" # 2 minutes timeout for pv deletion
-
     - name: Merge K-Bench results
       working-directory: k-bench
       shell: bash

.github/workflows/e2e-test-daily.yml

@@ -45,6 +45,7 @@ jobs:
           echo "res_group_name=$name" >> $GITHUB_OUTPUT
 
       - name: Run E2E test
+        id: e2e_test
         uses: ./.github/actions/e2e_test
         with:
           workerNodesCount: "2"
@@ -65,6 +66,8 @@ jobs:
         if: always()
         continue-on-error: true
         uses: ./.github/actions/constellation_destroy
+        with:
+          kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
 
       - name: Notify teams channel
         if: ${{ failure() && github.ref == 'refs/heads/main' }}

.github/workflows/e2e-test-manual-macos.yml

@@ -121,6 +121,7 @@ jobs:
         uses: google-github-actions/setup-gcloud@ee9693ff89cdf73862b8a13988f6a71070e8fc58 # tag=v0.6.2
 
       - name: Run manual E2E test
+        id: e2e_test
         uses: ./.github/actions/e2e_test
         with:
           workerNodesCount: ${{ github.event.inputs.workerNodesCount }}
@@ -145,6 +146,8 @@ jobs:
         if: always()
         continue-on-error: true
         uses: ./.github/actions/constellation_destroy
+        with:
+          cloudProvider: ${{ steps.e2e_test.outputs.kubeconfig }}
 
       - name: Notify teams channel
         if: ${{ failure() && github.ref == 'refs/heads/main' }}

.github/workflows/e2e-test-manual.yml

@@ -85,6 +85,7 @@ jobs:
           echo "res_group_name=$name" >> $GITHUB_OUTPUT
 
       - name: Run manual E2E test
+        id: e2e_test
         uses: ./.github/actions/e2e_test
         with:
           workerNodesCount: ${{ github.event.inputs.workerNodesCount }}
@@ -109,6 +110,8 @@ jobs:
         if: always()
         continue-on-error: true
         uses: ./.github/actions/constellation_destroy
+        with:
+          kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
 
       - name: Notify teams channel
         if: ${{ failure() && github.ref == 'refs/heads/main' }}

.github/workflows/e2e-test-weekly.yml

@@ -61,6 +61,7 @@ jobs:
           echo "res_group_name=$name" >> $GITHUB_OUTPUT
 
       - name: Run E2E test
+        id: e2e_test
         uses: ./.github/actions/e2e_test
         with:
           workerNodesCount: "2"
@@ -82,6 +83,8 @@ jobs:
         if: always()
         continue-on-error: true
         uses: ./.github/actions/constellation_destroy
+        with:
+          kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
 
       - name: Notify teams channel
         if: ${{ failure() && github.ref == 'refs/heads/main' }}