Adjust e2e benchmark to run on different attestation variants

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

.github/actions/e2e_benchmark/action.yml

@@ -5,7 +5,6 @@ inputs:
   cloudProvider:
     description: "Which cloud provider to use."
     required: true
-  # TODO: Create different report depending on the attestation variant
   attestationVariant:
     description: "Which attestation variant to use."
     required: true
@@ -56,18 +55,18 @@ runs:
         ref: 67c64c854841165b778979375444da1c02e02210
         path: k8s-bench-suite
 
-
-    - name: Run FIO benchmark without caching in Azure
-      if: inputs.cloudProvider == 'azure'
+    - name: Run FIO benchmark
       shell: bash
       env:
         KUBECONFIG: ${{ inputs.kubeconfig }}
       run: |
+        if [[ "${{ inputs.cloudProvider }}" == "azure" ]]
+        then
         cat <<EOF | kubectl apply -f -
         apiVersion: storage.k8s.io/v1
         kind: StorageClass
         metadata:
-          name: encrypted-rwo-no-cache
+          name: fio-benchmark
         allowVolumeExpansion: true
         allowedTopologies: []
         mountOptions: []
@@ -78,34 +77,31 @@ runs:
         reclaimPolicy: Delete
         volumeBindingMode: Immediate
         EOF
-        mkdir -p out
-        kubestr fio -e "out/fio-constellation-${{ inputs.cloudProvider }}.json" -o json -s encrypted-rwo-no-cache -z 400Gi -f .github/actions/e2e_benchmark/fio.ini
+        fi
 
-    - name: Run FIO benchmark
-      if: inputs.cloudProvider == 'gcp'
-      shell: bash
-      env:
-        KUBECONFIG: ${{ inputs.kubeconfig }}
-      run: |
+        if [[ "${{ inputs.cloudProvider }}" == "gcp" ]]
+        then
         cat <<EOF | kubectl apply -f -
         apiVersion: storage.k8s.io/v1
         kind: StorageClass
         metadata:
-          name: encrypted-balanced-rwo
+          name: fio-benchmark
         provisioner: gcp.csi.confidential.cloud
         volumeBindingMode: Immediate
         allowVolumeExpansion: true
         parameters:
           type: pd-balanced
         EOF
+        fi
+
         mkdir -p out
-        kubestr fio -e "out/fio-constellation-${{ inputs.cloudProvider }}.json" -o json -s encrypted-balanced-rwo -z 400Gi -f .github/actions/e2e_benchmark/fio.ini
+        kubestr fio -e "out/fio-constellation-${{ inputs.attestationVariant }}.json" -o json -s fio-benchmark -z 400Gi -f .github/actions/e2e_benchmark/fio.ini
 
     - name: Upload raw FIO benchmark results
       if: (!env.ACT)
       uses: ./.github/actions/artifact_upload
       with:
-        path: "out/fio-constellation-${{ inputs.cloudProvider }}.json"
+        path: "out/fio-constellation-${{ inputs.attestationVariant }}.json"
         name: "fio-constellation-${{ inputs.artifactNameSuffix }}.json"
         encryptionSecret: ${{ inputs.encryptionSecret }}
 
@@ -121,13 +117,13 @@ runs:
         echo "Server: $server"
         client="$(echo "$workers" | tail +2 | head -1 | cut -d '/' -f2)"
         echo "Client: $client"
-        k8s-bench-suite/knb -f "out/knb-constellation-${{ inputs.cloudProvider }}.json" -o json --server-node "$server" --client-node "$client"
+        k8s-bench-suite/knb -f "out/knb-constellation-${{ inputs.attestationVariant }}.json" -o json --server-node "$server" --client-node "$client"
 
     - name: Upload raw knb benchmark results
       if: (!env.ACT)
       uses: ./.github/actions/artifact_upload
       with:
-        path: "out/knb-constellation-${{ inputs.cloudProvider }}.json"
+        path: "out/knb-constellation-${{ inputs.attestationVariant }}.json"
         name: "knb-constellation-${{ inputs.artifactNameSuffix }}.json"
         encryptionSecret: ${{ inputs.encryptionSecret }}
 
@@ -139,6 +135,7 @@ runs:
         # Working directory containing the previous results as JSON and to contain the graphs
         BDIR: benchmarks
         CSP: ${{ inputs.cloudProvider }}
+        ATTESTATION_VARIANT: ${{ inputs.attestationVariant }}
       run: |
         mkdir -p benchmarks
         python .github/actions/e2e_benchmark/evaluate/parse.py
@@ -148,7 +145,7 @@ runs:
       uses: ./.github/actions/artifact_upload
       with:
         path: >
-          benchmarks/constellation-${{ inputs.cloudProvider }}.json
+          benchmarks/constellation-${{ inputs.attestationVariant }}.json
         name: "benchmarks-${{ inputs.artifactNameSuffix }}"
         encryptionSecret: ${{ inputs.encryptionSecret }}
 
@@ -166,12 +163,10 @@ runs:
 
     - name: Get previous benchmark records from S3
       shell: bash
-      env:
-        CSP: ${{ inputs.cloudProvider }}
       run: |
-        aws s3 cp --recursive ${S3_PATH} ./ --no-progress
-        if [[ -f constellation-${CSP}.json ]]; then
-          mv constellation-${CSP}.json benchmarks/constellation-${CSP}-previous.json
+        aws s3 cp ${S3_PATH}/constellation-${{ inputs.attestationVariant }}.json ./ --no-progress || true # ignore if the file doesn't exist
+        if [[ -f constellation-${{ inputs.attestationVariant }}.json ]]; then
+          mv constellation-${{ inputs.attestationVariant }}.json benchmarks/constellation-${{ inputs.attestationVariant }}-previous.json
         else
             echo "::warning::Couldn't retrieve previous benchmark records from s3"
         fi
@@ -180,15 +175,15 @@ runs:
       shell: bash
       env:
         # Paths to benchmark results as JSON of the previous run and the current run
-        PREV_BENCH: benchmarks/constellation-${{ inputs.cloudProvider }}-previous.json
-        CURR_BENCH: benchmarks/constellation-${{ inputs.cloudProvider }}.json
+        PREV_BENCH: benchmarks/constellation-${{ inputs.attestationVariant }}-previous.json
+        CURR_BENCH: benchmarks/constellation-${{ inputs.attestationVariant }}.json
       run: |
         if [[ -f "$PREV_BENCH" ]]; then
           # Fails if the results are outside the threshold range
           python .github/actions/e2e_benchmark/evaluate/compare.py >> $GITHUB_STEP_SUMMARY
         fi
 
-    - name: Upload benchmark results to opensearch
+    - name: Upload benchmark results to OpenSearch
       if: (!env.ACT)
       shell: bash
       env:
@@ -198,14 +193,12 @@ runs:
       run: |
         curl -XPOST \
             -u "${OPENSEARCH_USER}:${OPENSEARCH_PWD}" \
-            "${OPENSEARCH_DOMAIN}/benchmarks-${{ inputs.cloudProvider }}-$(date '+%Y')"/_doc \
-            --data-binary @benchmarks/constellation-${{ inputs.cloudProvider }}.json \
+            "${OPENSEARCH_DOMAIN}/benchmarks-${{ inputs.attestationVariant }}-$(date '+%Y')"/_doc \
+            --data-binary @benchmarks/constellation-${{ inputs.attestationVariant }}.json \
             -H 'Content-Type: application/json'
 
     - name: Update benchmark records in S3
       if: github.ref_name == 'main'
-      shell: bash
-      env:
-        CSP: ${{ inputs.cloudProvider }}
+      shell: bas
       run: |
-        aws s3 cp benchmarks/constellation-${CSP}.json ${S3_PATH}/constellation-${CSP}.json
+        aws s3 cp benchmarks/constellation-${attestationVariant}.json ${S3_PATH}/constellation-${attestationVariant}.json

.github/actions/e2e_benchmark/evaluate/compare.py

@@ -94,18 +94,18 @@ class BenchmarkComparer:
             raise ValueError('Failed reading benchmark file: {e}'.format(e=e))
 
         try:
-            name = bench_curr['provider']
+            name = bench_curr['attestationVariant']
         except KeyError:
             raise ValueError(
-                'Current benchmark record file does not contain provider.')
+                'Current benchmark record file does not contain attestationVariant.')
         try:
-            prev_name = bench_prev['provider']
+            prev_name = bench_prev['attestationVariant']
         except KeyError:
             raise ValueError(
-                'Previous benchmark record file does not contain provider.')
+                'Previous benchmark record file does not contain attestationVariant.')
         if name != prev_name:
             raise ValueError(
-                'Cloud providers of previous and current benchmark data do not match.')
+                'Cloud attestationVariants of previous and current benchmark data do not match.')
 
         if 'fio' not in bench_prev.keys() or 'fio' not in bench_curr.keys():
             raise ValueError('Benchmarks do not both contain fio records.')

.github/actions/e2e_benchmark/evaluate/parse.py

@@ -7,7 +7,7 @@ from datetime import datetime
 from evaluators import fio, knb
 
 
-def configure() -> Tuple[str, str, str, str | None, str, str, str, str]:
+def configure() -> Tuple[str, str, str, str, str | None, str, str, str, str]:
     """Read the benchmark data paths.
 
     Expects ENV vars (required):
@@ -25,27 +25,29 @@ def configure() -> Tuple[str, str, str, str | None, str, str, str, str]:
     """
     base_path = os.environ.get('BENCH_RESULTS', None)
     csp = os.environ.get('CSP', None)
+    attestation_variant = os.environ.get('ATTESTATION_VARIANT', None)
     out_dir = os.environ.get('BDIR', None)
-    if not base_path or not csp or not out_dir:
+    if not base_path or not csp or not out_dir or not attestation_variant:
         raise TypeError(
-            'ENV variables BENCH_RESULTS, CSP, BDIR are required.')
+            'ENV variables BENCH_RESULTS, CSP, BDIR, ATTESTATION_VARIANT are required.')
 
     ext_provider_name = os.environ.get('EXT_NAME', None)
     commit_hash = os.environ.get('GITHUB_SHA', 'N/A')
     commit_ref = os.environ.get('GITHUB_REF_NAME', 'N/A')
     actor = os.environ.get('GITHUB_ACTOR', 'N/A')
     workflow = os.environ.get('GITHUB_WORKFLOW', 'N/A')
-    return base_path, csp, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow
+    return base_path, csp, attestation_variant, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow
 
 
 class BenchmarkParser:
-    def __init__(self, base_path, csp, out_dir, ext_provider_name=None, commit_hash="N/A", commit_ref="N/A", actor="N/A", workflow="N/A"):
+    def __init__(self, base_path, csp, attestation_variant, out_dir, ext_provider_name=None, commit_hash="N/A", commit_ref="N/A", actor="N/A", workflow="N/A"):
         self.base_path = base_path
         self.csp = csp
+        self.attestation_variant = attestation_variant
         self.out_dir = out_dir
         self.ext_provider_name = ext_provider_name
         if not self.ext_provider_name:
-            self.ext_provider_name = f'constellation-{csp}'
+            self.ext_provider_name = f'constellation-{attestation_variant}'
         self.commit_hash = commit_hash
         self.commit_ref = commit_ref
         self.actor = actor
@@ -88,6 +90,7 @@ class BenchmarkParser:
         },
             '@timestamp': str(timestamp),
             'provider': self.ext_provider_name,
+            'attestationVariant': self.attestation_variant,
             'fio': {},
             'knb': {}}
 
@@ -101,8 +104,8 @@ class BenchmarkParser:
 
 
 def main():
-    base_path, csp, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow = configure()
-    p = BenchmarkParser(base_path, csp, out_dir, ext_provider_name,
+    base_path, csp, attestation_variant, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow = configure()
+    p = BenchmarkParser(base_path, csp, attestation_variant, out_dir, ext_provider_name,
                         commit_hash, commit_ref, actor, workflow)
     p.parse()