From fc9b49da27edbe4a2b54a19e297eabfc67ab8155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Wei=C3=9Fe?= Date: Wed, 29 May 2024 09:23:35 +0200 Subject: [PATCH] Adjust e2e benchmark to run on different attestation variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Weiße --- .github/actions/e2e_benchmark/action.yml | 59 ++++++++----------- .../actions/e2e_benchmark/evaluate/compare.py | 10 ++-- .../actions/e2e_benchmark/evaluate/parse.py | 19 +++--- 3 files changed, 42 insertions(+), 46 deletions(-) diff --git a/.github/actions/e2e_benchmark/action.yml b/.github/actions/e2e_benchmark/action.yml index 14524ca91..90fea832c 100644 --- a/.github/actions/e2e_benchmark/action.yml +++ b/.github/actions/e2e_benchmark/action.yml @@ -5,7 +5,6 @@ inputs: cloudProvider: description: "Which cloud provider to use." required: true - # TODO: Create different report depending on the attestation variant attestationVariant: description: "Which attestation variant to use." required: true @@ -56,18 +55,18 @@ runs: ref: 67c64c854841165b778979375444da1c02e02210 path: k8s-bench-suite - - - name: Run FIO benchmark without caching in Azure - if: inputs.cloudProvider == 'azure' + - name: Run FIO benchmark shell: bash env: KUBECONFIG: ${{ inputs.kubeconfig }} run: | + if [[ "${{ inputs.cloudProvider }}" == "azure" ]] + then cat < - benchmarks/constellation-${{ inputs.cloudProvider }}.json + benchmarks/constellation-${{ inputs.attestationVariant }}.json name: "benchmarks-${{ inputs.artifactNameSuffix }}" encryptionSecret: ${{ inputs.encryptionSecret }} @@ -166,12 +163,10 @@ runs: - name: Get previous benchmark records from S3 shell: bash - env: - CSP: ${{ inputs.cloudProvider }} run: | - aws s3 cp --recursive ${S3_PATH} ./ --no-progress - if [[ -f constellation-${CSP}.json ]]; then - mv constellation-${CSP}.json benchmarks/constellation-${CSP}-previous.json + aws s3 cp ${S3_PATH}/constellation-${{ inputs.attestationVariant }}.json ./ --no-progress || true # ignore if the file doesn't exist + if [[ -f constellation-${{ inputs.attestationVariant }}.json ]]; then + mv constellation-${{ inputs.attestationVariant }}.json benchmarks/constellation-${{ inputs.attestationVariant }}-previous.json else echo "::warning::Couldn't retrieve previous benchmark records from s3" fi @@ -180,15 +175,15 @@ runs: shell: bash env: # Paths to benchmark results as JSON of the previous run and the current run - PREV_BENCH: benchmarks/constellation-${{ inputs.cloudProvider }}-previous.json - CURR_BENCH: benchmarks/constellation-${{ inputs.cloudProvider }}.json + PREV_BENCH: benchmarks/constellation-${{ inputs.attestationVariant }}-previous.json + CURR_BENCH: benchmarks/constellation-${{ inputs.attestationVariant }}.json run: | if [[ -f "$PREV_BENCH" ]]; then # Fails if the results are outside the threshold range python .github/actions/e2e_benchmark/evaluate/compare.py >> $GITHUB_STEP_SUMMARY fi - - name: Upload benchmark results to opensearch + - name: Upload benchmark results to OpenSearch if: (!env.ACT) shell: bash env: @@ -198,14 +193,12 @@ runs: run: | curl -XPOST \ -u "${OPENSEARCH_USER}:${OPENSEARCH_PWD}" \ - "${OPENSEARCH_DOMAIN}/benchmarks-${{ inputs.cloudProvider }}-$(date '+%Y')"/_doc \ - --data-binary @benchmarks/constellation-${{ inputs.cloudProvider }}.json \ + "${OPENSEARCH_DOMAIN}/benchmarks-${{ inputs.attestationVariant }}-$(date '+%Y')"/_doc \ + --data-binary @benchmarks/constellation-${{ inputs.attestationVariant }}.json \ -H 'Content-Type: application/json' - name: Update benchmark records in S3 if: github.ref_name == 'main' - shell: bash - env: - CSP: ${{ inputs.cloudProvider }} + shell: bas run: | - aws s3 cp benchmarks/constellation-${CSP}.json ${S3_PATH}/constellation-${CSP}.json + aws s3 cp benchmarks/constellation-${attestationVariant}.json ${S3_PATH}/constellation-${attestationVariant}.json diff --git a/.github/actions/e2e_benchmark/evaluate/compare.py b/.github/actions/e2e_benchmark/evaluate/compare.py index 59d10a186..87faac09e 100644 --- a/.github/actions/e2e_benchmark/evaluate/compare.py +++ b/.github/actions/e2e_benchmark/evaluate/compare.py @@ -94,18 +94,18 @@ class BenchmarkComparer: raise ValueError('Failed reading benchmark file: {e}'.format(e=e)) try: - name = bench_curr['provider'] + name = bench_curr['attestationVariant'] except KeyError: raise ValueError( - 'Current benchmark record file does not contain provider.') + 'Current benchmark record file does not contain attestationVariant.') try: - prev_name = bench_prev['provider'] + prev_name = bench_prev['attestationVariant'] except KeyError: raise ValueError( - 'Previous benchmark record file does not contain provider.') + 'Previous benchmark record file does not contain attestationVariant.') if name != prev_name: raise ValueError( - 'Cloud providers of previous and current benchmark data do not match.') + 'Cloud attestationVariants of previous and current benchmark data do not match.') if 'fio' not in bench_prev.keys() or 'fio' not in bench_curr.keys(): raise ValueError('Benchmarks do not both contain fio records.') diff --git a/.github/actions/e2e_benchmark/evaluate/parse.py b/.github/actions/e2e_benchmark/evaluate/parse.py index fedce5c70..8d9353343 100644 --- a/.github/actions/e2e_benchmark/evaluate/parse.py +++ b/.github/actions/e2e_benchmark/evaluate/parse.py @@ -7,7 +7,7 @@ from datetime import datetime from evaluators import fio, knb -def configure() -> Tuple[str, str, str, str | None, str, str, str, str]: +def configure() -> Tuple[str, str, str, str, str | None, str, str, str, str]: """Read the benchmark data paths. Expects ENV vars (required): @@ -25,27 +25,29 @@ def configure() -> Tuple[str, str, str, str | None, str, str, str, str]: """ base_path = os.environ.get('BENCH_RESULTS', None) csp = os.environ.get('CSP', None) + attestation_variant = os.environ.get('ATTESTATION_VARIANT', None) out_dir = os.environ.get('BDIR', None) - if not base_path or not csp or not out_dir: + if not base_path or not csp or not out_dir or not attestation_variant: raise TypeError( - 'ENV variables BENCH_RESULTS, CSP, BDIR are required.') + 'ENV variables BENCH_RESULTS, CSP, BDIR, ATTESTATION_VARIANT are required.') ext_provider_name = os.environ.get('EXT_NAME', None) commit_hash = os.environ.get('GITHUB_SHA', 'N/A') commit_ref = os.environ.get('GITHUB_REF_NAME', 'N/A') actor = os.environ.get('GITHUB_ACTOR', 'N/A') workflow = os.environ.get('GITHUB_WORKFLOW', 'N/A') - return base_path, csp, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow + return base_path, csp, attestation_variant, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow class BenchmarkParser: - def __init__(self, base_path, csp, out_dir, ext_provider_name=None, commit_hash="N/A", commit_ref="N/A", actor="N/A", workflow="N/A"): + def __init__(self, base_path, csp, attestation_variant, out_dir, ext_provider_name=None, commit_hash="N/A", commit_ref="N/A", actor="N/A", workflow="N/A"): self.base_path = base_path self.csp = csp + self.attestation_variant = attestation_variant self.out_dir = out_dir self.ext_provider_name = ext_provider_name if not self.ext_provider_name: - self.ext_provider_name = f'constellation-{csp}' + self.ext_provider_name = f'constellation-{attestation_variant}' self.commit_hash = commit_hash self.commit_ref = commit_ref self.actor = actor @@ -88,6 +90,7 @@ class BenchmarkParser: }, '@timestamp': str(timestamp), 'provider': self.ext_provider_name, + 'attestationVariant': self.attestation_variant, 'fio': {}, 'knb': {}} @@ -101,8 +104,8 @@ class BenchmarkParser: def main(): - base_path, csp, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow = configure() - p = BenchmarkParser(base_path, csp, out_dir, ext_provider_name, + base_path, csp, attestation_variant, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow = configure() + p = BenchmarkParser(base_path, csp, attestation_variant, out_dir, ext_provider_name, commit_hash, commit_ref, actor, workflow) p.parse()