constellation/.github/actions/e2e_benchmark/action.yml

name: benchmark
description: "Run benchmarks"

inputs:
  cloudProvider:
    description: "Which cloud provider to use."
    required: true
  # TODO: Create different report depending on the attestation variant
  attestationVariant:
    description: "Which attestation variant to use."
    required: true
  kubeconfig:
    description: "The kubeconfig of the cluster to test."
    required: true
  awsOpenSearchDomain:
    description: "AWS OpenSearch Endpoint Domain to upload the results."
    required: false
  awsOpenSearchUsers:
    description: "AWS OpenSearch User to upload the results."
    required: false
  awsOpenSearchPwd:
    description: "AWS OpenSearch Password to upload the results."
    required: false
  artifactNameSuffix:
    description: "Suffix for artifact naming."
    required: true
  encryptionSecret:
    description: 'The secret to use for encrypting the artifact.'
    required: true

runs:
  using: "composite"

  steps:
    - name: Setup python
      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
      with:
        python-version: "3.10"

    - name: Install kubestr
      shell: bash
      env:
        KUBESTR_VER: "0.4.37"
      run: |
        HOSTOS="$(go env GOOS)"
        HOSTARCH="$(go env GOARCH)"
        curl -fsSLO https://github.com/kastenhq/kubestr/releases/download/v${KUBESTR_VER}/kubestr_${KUBESTR_VER}_${HOSTOS}_${HOSTARCH}.tar.gz
        tar -xzf kubestr_${KUBESTR_VER}_${HOSTOS}_${HOSTARCH}.tar.gz
        install kubestr /usr/local/bin        

    - name: Checkout k8s-bench-suite
      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
      with:
        fetch-depth: 0
        repository: "edgelesssys/k8s-bench-suite"
        ref: 67c64c854841165b778979375444da1c02e02210
        path: k8s-bench-suite


    - name: Run FIO benchmark without caching in Azure
      if: inputs.cloudProvider == 'azure'
      shell: bash
      env:
        KUBECONFIG: ${{ inputs.kubeconfig }}
      run: |
        cat <<EOF | kubectl apply -f -
        apiVersion: storage.k8s.io/v1
        kind: StorageClass
        metadata:
          name: encrypted-rwo-no-cache
        allowVolumeExpansion: true
        allowedTopologies: []
        mountOptions: []
        parameters:
          skuname: StandardSSD_LRS
          cachingMode: None
        provisioner: azuredisk.csi.confidential.cloud
        reclaimPolicy: Delete
        volumeBindingMode: Immediate
        EOF
        mkdir -p out
        kubestr fio -e "out/fio-constellation-${{ inputs.cloudProvider }}.json" -o json -s encrypted-rwo-no-cache -z 400Gi -f .github/actions/e2e_benchmark/fio.ini        

    - name: Run FIO benchmark
      if: inputs.cloudProvider == 'gcp'
      shell: bash
      env:
        KUBECONFIG: ${{ inputs.kubeconfig }}
      run: |
        cat <<EOF | kubectl apply -f -
        apiVersion: storage.k8s.io/v1
        kind: StorageClass
        metadata:
          name: encrypted-balanced-rwo
        provisioner: gcp.csi.confidential.cloud
        volumeBindingMode: Immediate
        allowVolumeExpansion: true
        parameters:
          type: pd-balanced
        EOF
        mkdir -p out
        kubestr fio -e "out/fio-constellation-${{ inputs.cloudProvider }}.json" -o json -s encrypted-balanced-rwo -z 400Gi -f .github/actions/e2e_benchmark/fio.ini        

    - name: Upload raw FIO benchmark results
      if: (!env.ACT)
      uses: ./.github/actions/artifact_upload
      with:
        path: "out/fio-constellation-${{ inputs.cloudProvider }}.json"
        name: "fio-constellation-${{ inputs.artifactNameSuffix }}.json"
        encryptionSecret: ${{ inputs.encryptionSecret }}

    - name: Run knb benchmark
      shell: bash
      env:
        KUBECONFIG: ${{ inputs.kubeconfig }}
        TERM: xterm-256color
      run: |
        workers="$(kubectl get nodes -o name | grep worker)"
        echo -e "Found workers:\n$workers"
        server="$(echo "$workers" | tail +1 | head -1 | cut -d '/' -f2)"
        echo "Server: $server"
        client="$(echo "$workers" | tail +2 | head -1 | cut -d '/' -f2)"
        echo "Client: $client"
        k8s-bench-suite/knb -f "out/knb-constellation-${{ inputs.cloudProvider }}.json" -o json --server-node "$server" --client-node "$client"        

    - name: Upload raw knb benchmark results
      if: (!env.ACT)
      uses: ./.github/actions/artifact_upload
      with:
        path: "out/knb-constellation-${{ inputs.cloudProvider }}.json"
        name: "knb-constellation-${{ inputs.artifactNameSuffix }}.json"
        encryptionSecret: ${{ inputs.encryptionSecret }}

    - name: Parse results, create diagrams and post the progression summary
      shell: bash
      env:
        # Original result directory
        BENCH_RESULTS: out/
        # Working directory containing the previous results as JSON and to contain the graphs
        BDIR: benchmarks
        CSP: ${{ inputs.cloudProvider }}
      run: |
        mkdir -p benchmarks
        python .github/actions/e2e_benchmark/evaluate/parse.py        

    - name: Upload benchmark results to action run
      if: (!env.ACT)
      uses: ./.github/actions/artifact_upload
      with:
        path: >
          benchmarks/constellation-${{ inputs.cloudProvider }}.json          
        name: "benchmarks-${{ inputs.artifactNameSuffix }}"
        encryptionSecret: ${{ inputs.encryptionSecret }}

    - name: Assume AWS role to retrieve and update benchmarks in S3
      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
      with:
        role-to-assume: arn:aws:iam::795746500882:role/GithubActionUpdateBenchmarks
        aws-region: us-east-2

    - name: Set S3 artifact store
      shell: bash
      env:
        ARTIFACT_BUCKET_CONSTELLATION: "edgeless-artifact-store/constellation"
      run: echo S3_PATH=s3://${ARTIFACT_BUCKET_CONSTELLATION}/benchmarks >> $GITHUB_ENV

    - name: Get previous benchmark records from S3
      shell: bash
      env:
        CSP: ${{ inputs.cloudProvider }}
      run: |
        aws s3 cp --recursive ${S3_PATH} ./ --no-progress
        if [[ -f constellation-${CSP}.json ]]; then
          mv constellation-${CSP}.json benchmarks/constellation-${CSP}-previous.json
        else
            echo "::warning::Couldn't retrieve previous benchmark records from s3"
        fi        

    - name: Compare results
      shell: bash
      env:
        # Paths to benchmark results as JSON of the previous run and the current run
        PREV_BENCH: benchmarks/constellation-${{ inputs.cloudProvider }}-previous.json
        CURR_BENCH: benchmarks/constellation-${{ inputs.cloudProvider }}.json
      run: |
        if [[ -f "$PREV_BENCH" ]]; then
          # Fails if the results are outside the threshold range
          python .github/actions/e2e_benchmark/evaluate/compare.py >> $GITHUB_STEP_SUMMARY
        fi        

    - name: Upload benchmark results to opensearch
      if: (!env.ACT)
      shell: bash
      env:
        OPENSEARCH_DOMAIN: ${{ inputs.awsOpenSearchDomain }}
        OPENSEARCH_USER: ${{ inputs.awsOpenSearchUsers }}
        OPENSEARCH_PWD: ${{ inputs.awsOpenSearchPwd }}
      run: |
        curl -XPOST \
            -u "${OPENSEARCH_USER}:${OPENSEARCH_PWD}" \
            "${OPENSEARCH_DOMAIN}/benchmarks-${{ inputs.cloudProvider }}-$(date '+%Y')"/_doc \
            --data-binary @benchmarks/constellation-${{ inputs.cloudProvider }}.json \
            -H 'Content-Type: application/json'        

    - name: Update benchmark records in S3
      if: github.ref_name == 'main'
      shell: bash
      env:
        CSP: ${{ inputs.cloudProvider }}
      run: |
        aws s3 cp benchmarks/constellation-${CSP}.json ${S3_PATH}/constellation-${CSP}.json