mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-09-20 00:06:21 +00:00
c020f7ac20
* ci: improve constellation_create error message When we hit a timeout due to nodes not coming up, the actual error message is hard to make out because it's buried in a group. With the right formatting, the error message will be highlighted in the UI. Another improvement is to output the state of nodes, which helps debugging the cause of nodes not joining or not becoming ready. * cleanup: use NodeVersionResourceName constant ... instead of literal strings. * ci: correctly notify on e2e upgrade error * atls: report cert extension OIDs on mismatch If the certificate contains an attestation document for SEV-SNP, but the given validator is for Nitro, verifyEmbeddedReport should not claim that there is no attestation document, but that there is no _compatible_ one and what the incompatible ones were.
537 lines
20 KiB
YAML
537 lines
20 KiB
YAML
name: e2e test upgrade
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
attestationVariant:
|
|
description: "Which attestation variant to use."
|
|
type: choice
|
|
options:
|
|
- "gcp-sev-es"
|
|
- "azure-sev-snp"
|
|
- "azure-tdx"
|
|
- "aws-sev-snp"
|
|
default: "azure-sev-snp"
|
|
required: true
|
|
nodeCount:
|
|
description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
|
|
default: "3:2"
|
|
type: string
|
|
fromVersion:
|
|
description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
|
|
type: string
|
|
required: true
|
|
gitRef:
|
|
description: Ref to build upgrading CLI on, empty for HEAD.
|
|
type: string
|
|
default: "head"
|
|
required: false
|
|
toImage:
|
|
description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
|
|
type: string
|
|
required: false
|
|
toKubernetes:
|
|
description: Kubernetes version to target for the upgrade, empty for target's default version.
|
|
type: string
|
|
required: false
|
|
toMicroservices:
|
|
description: Microservice version to target for the upgrade, empty for target's default version.
|
|
type: string
|
|
required: false
|
|
simulatedTargetVersion:
|
|
description: Enter a version to build the CLI with. This can be used to simulate a patch-upgrade.
|
|
type: string
|
|
required: false
|
|
regionZone:
|
|
description: "Region or zone to create the cluster in. Leave empty for default region/zone."
|
|
type: string
|
|
workflow_call:
|
|
inputs:
|
|
attestationVariant:
|
|
description: "Which attestation variant to use."
|
|
type: string
|
|
required: true
|
|
nodeCount:
|
|
description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
|
|
default: "3:2"
|
|
type: string
|
|
fromVersion:
|
|
description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
|
|
type: string
|
|
required: true
|
|
gitRef:
|
|
description: Ref to build upgrading CLI on.
|
|
type: string
|
|
default: "head"
|
|
required: false
|
|
toImage:
|
|
description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
|
|
type: string
|
|
required: false
|
|
toKubernetes:
|
|
description: Kubernetes version to target for the upgrade, empty for target's default version.
|
|
type: string
|
|
required: false
|
|
toMicroservices:
|
|
description: Kubernetes version to target for the upgrade, empty for target's default version.
|
|
type: string
|
|
required: false
|
|
simulatedTargetVersion:
|
|
description: Enter a version to build the CLI with. This can be used to simulate a patch-upgrade.
|
|
type: string
|
|
required: false
|
|
scheduled:
|
|
description: Whether this is a scheduled run.
|
|
type: boolean
|
|
default: false
|
|
required: false
|
|
|
|
jobs:
|
|
generate-input-parameters:
|
|
name: Generate input parameters
|
|
runs-on: ubuntu-22.04
|
|
permissions:
|
|
id-token: write
|
|
contents: read
|
|
outputs:
|
|
workerNodes: ${{ steps.split-nodeCount.outputs.workerNodes }}
|
|
controlPlaneNodes: ${{ steps.split-nodeCount.outputs.controlPlaneNodes }}
|
|
cloudProvider: ${{ steps.split-attestationVariant.outputs.cloudProvider }}
|
|
steps:
|
|
- name: Split nodeCount
|
|
id: split-nodeCount
|
|
shell: bash
|
|
run: |
|
|
nodeCount="${{ inputs.nodeCount }}"
|
|
workerNodes="${nodeCount##*:}"
|
|
controlPlaneNodes="${nodeCount%%:*}"
|
|
|
|
if [[ -z "${workerNodes}" ]] || [[ -z "{controlPlaneNodes}" ]]; then
|
|
echo "Invalid nodeCount input: '${nodeCount}'."
|
|
exit 1
|
|
fi
|
|
|
|
echo "workerNodes=${workerNodes}" | tee -a "$GITHUB_OUTPUT"
|
|
echo "controlPlaneNodes=${controlPlaneNodes}" | tee -a "$GITHUB_OUTPUT"
|
|
|
|
- name: Split attestationVariant
|
|
id: split-attestationVariant
|
|
shell: bash
|
|
run: |
|
|
attestationVariant="${{ inputs.attestationVariant }}"
|
|
cloudProvider="${attestationVariant%%-*}"
|
|
|
|
echo "cloudProvider=${cloudProvider}" | tee -a "$GITHUB_OUTPUT"
|
|
|
|
build-target-cli:
|
|
name: Build upgrade target version CLI
|
|
runs-on: ubuntu-22.04
|
|
permissions:
|
|
id-token: write
|
|
checks: write
|
|
contents: read
|
|
packages: write
|
|
steps:
|
|
- name: Checkout
|
|
if: inputs.gitRef == 'head'
|
|
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
|
|
|
|
- name: Checkout ref
|
|
if: inputs.gitRef != 'head'
|
|
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ inputs.gitRef }}
|
|
|
|
- name: Setup Bazel & Nix
|
|
uses: ./.github/actions/setup_bazel_nix
|
|
with:
|
|
useCache: "true"
|
|
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
|
|
|
|
- name: Log in to the Container registry
|
|
uses: ./.github/actions/container_registry_login
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ github.actor }}
|
|
password: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Simulate patch upgrade
|
|
if: inputs.simulatedTargetVersion != ''
|
|
run: |
|
|
echo ${{ inputs.simulatedTargetVersion }} > version.txt
|
|
|
|
- name: Build CLI
|
|
uses: ./.github/actions/build_cli
|
|
with:
|
|
enterpriseCLI: true
|
|
outputPath: "build/constellation"
|
|
push: true
|
|
|
|
- name: Upload CLI binary
|
|
uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
|
|
with:
|
|
name: constellation
|
|
path: build/constellation
|
|
|
|
create-cluster:
|
|
name: Create upgrade origin version cluster
|
|
runs-on: ubuntu-22.04
|
|
permissions:
|
|
id-token: write
|
|
checks: write
|
|
contents: read
|
|
packages: write
|
|
needs: [generate-input-parameters]
|
|
outputs:
|
|
kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
|
|
steps:
|
|
- name: Checkout
|
|
if: inputs.gitRef == 'head'
|
|
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
|
|
|
|
- name: Checkout ref
|
|
if: inputs.gitRef != 'head'
|
|
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ inputs.gitRef }}
|
|
|
|
- uses: ./.github/actions/setup_bazel_nix
|
|
with:
|
|
useCache: "true"
|
|
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
|
|
|
|
- name: Create cluster with 'fromVersion' CLI.
|
|
id: e2e_test
|
|
uses: ./.github/actions/e2e_test
|
|
with:
|
|
workerNodesCount: ${{ needs.generate-input-parameters.outputs.workerNodes }}
|
|
controlNodesCount: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }}
|
|
cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
|
|
attestationVariant: ${{ inputs.attestationVariant }}
|
|
osImage: ${{ inputs.fromVersion }}
|
|
isDebugImage: "false"
|
|
cliVersion: ${{ inputs.fromVersion }}
|
|
regionZone: ${{ inputs.regionZone }}
|
|
gcpProject: constellation-e2e
|
|
gcpClusterCreateServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"
|
|
gcpIAMCreateServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
|
|
test: "upgrade"
|
|
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
|
|
azureClusterCreateCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
|
|
azureIAMCreateCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
|
|
registry: ghcr.io
|
|
githubToken: ${{ secrets.GITHUB_TOKEN }}
|
|
awsOpenSearchDomain: ${{ secrets.AWS_OPENSEARCH_DOMAIN }}
|
|
awsOpenSearchUsers: ${{ secrets.AWS_OPENSEARCH_USER }}
|
|
awsOpenSearchPwd: ${{ secrets.AWS_OPENSEARCH_PWD }}
|
|
clusterCreation: "cli"
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
- name: Remove Terraform plugin cache
|
|
if: always()
|
|
run: |
|
|
rm -rf constellation-terraform/.terraform
|
|
rm -rf constellation-iam-terraform/.terraform
|
|
|
|
- name: Upload Working Directory
|
|
if: always()
|
|
uses: ./.github/actions/artifact_upload
|
|
with:
|
|
name: constellation-pre-test
|
|
path: >
|
|
${{ steps.e2e_test.outputs.kubeconfig }}
|
|
constellation-terraform
|
|
constellation-iam-terraform
|
|
constellation-conf.yaml
|
|
constellation-state.yaml
|
|
constellation-mastersecret.json
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
- name: Upload SA Key
|
|
if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'gcp'
|
|
uses: ./.github/actions/artifact_upload
|
|
with:
|
|
name: sa-key
|
|
path: >
|
|
gcpServiceAccountKey.json
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
e2e-upgrade:
|
|
name: Run upgrade test
|
|
runs-on: ubuntu-22.04
|
|
permissions:
|
|
id-token: write
|
|
checks: write
|
|
contents: read
|
|
packages: write
|
|
needs:
|
|
- generate-input-parameters
|
|
- build-target-cli
|
|
- create-cluster
|
|
steps:
|
|
- name: Checkout
|
|
if: inputs.gitRef == 'head'
|
|
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
|
|
|
|
- name: Checkout ref
|
|
if: inputs.gitRef != 'head'
|
|
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ inputs.gitRef }}
|
|
|
|
- name: Setup Bazel & Nix
|
|
uses: ./.github/actions/setup_bazel_nix
|
|
with:
|
|
useCache: "true"
|
|
buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
|
|
|
|
- name: Login to AWS
|
|
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
|
|
with:
|
|
role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
|
|
aws-region: eu-central-1
|
|
|
|
- name: Find latest nightly image
|
|
id: find-image
|
|
if: inputs.toImage == ''
|
|
uses: ./.github/actions/versionsapi
|
|
with:
|
|
command: latest
|
|
ref: main
|
|
stream: nightly
|
|
|
|
- name: Login to GCP (IAM service account)
|
|
if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp'
|
|
uses: ./.github/actions/login_gcp
|
|
with:
|
|
service_account: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
|
|
|
|
- name: Login to AWS (IAM role)
|
|
if: needs.generate-input-parameters.outputs.cloudProvider == 'aws'
|
|
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
|
|
with:
|
|
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2EIAM
|
|
aws-region: eu-central-1
|
|
# extend token expiry to 6 hours to ensure constellation can terminate
|
|
role-duration-seconds: 21600
|
|
|
|
- name: Login to Azure (IAM service principal)
|
|
if: needs.generate-input-parameters.outputs.cloudProvider == 'azure'
|
|
uses: ./.github/actions/login_azure
|
|
with:
|
|
azure_credentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
|
|
|
|
- name: Download CLI
|
|
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
|
|
with:
|
|
name: constellation
|
|
path: build
|
|
|
|
- name: Download Working Directory (Pre-test)
|
|
uses: ./.github/actions/artifact_download
|
|
with:
|
|
name: constellation-pre-test
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
- name: Download SA Key
|
|
if: needs.generate-input-parameters.outputs.cloudProvider == 'gcp'
|
|
uses: ./.github/actions/artifact_download
|
|
with:
|
|
name: sa-key
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
- name: Make Constellation executable and add to PATH
|
|
if: always()
|
|
run: |
|
|
chmod +x build/constellation
|
|
export PATH="$PATH:build"
|
|
echo "build" >> "$GITHUB_PATH"
|
|
|
|
- name: Migrate config
|
|
id: constellation-config-migrate
|
|
run: |
|
|
./build/constellation config migrate --debug
|
|
|
|
- name: Upgrade IAM configuration
|
|
id: constellation-iam-upgrade
|
|
uses: ./.github/actions/constellation_iam_upgrade
|
|
|
|
- name: Login to GCP (Cluster service account)
|
|
if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'gcp'
|
|
uses: ./.github/actions/login_gcp
|
|
with:
|
|
service_account: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"
|
|
|
|
- name: Login to AWS (Cluster role)
|
|
if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'aws'
|
|
uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1
|
|
with:
|
|
role-to-assume: arn:aws:iam::795746500882:role/GithubActionsE2ECluster
|
|
aws-region: eu-central-1
|
|
# extend token expiry to 6 hours to ensure constellation can terminate
|
|
role-duration-seconds: 21600
|
|
|
|
- name: Login to Azure (Cluster service principal)
|
|
if: always() && needs.generate-input-parameters.outputs.cloudProvider == 'azure'
|
|
uses: ./.github/actions/login_azure
|
|
with:
|
|
azure_credentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
|
|
|
|
- name: Run upgrade test
|
|
env:
|
|
KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }}
|
|
IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }}
|
|
KUBERNETES: ${{ inputs.toKubernetes }}
|
|
MICROSERVICES: ${{ inputs.toMicroservices }}
|
|
WORKERNODES: ${{ needs.generate-input-parameters.outputs.workerNodes }}
|
|
CONTROLNODES: ${{ needs.generate-input-parameters.outputs.controlPlaneNodes }}
|
|
run: |
|
|
echo "Image target: $IMAGE"
|
|
echo "K8s target: $KUBERNETES"
|
|
echo "Microservice target: $MICROSERVICES"
|
|
|
|
if [[ -n ${MICROSERVICES} ]]; then
|
|
MICROSERVICES_FLAG="--target-microservices=$MICROSERVICES"
|
|
fi
|
|
if [[ -n ${KUBERNETES} ]]; then
|
|
KUBERNETES_FLAG="--target-kubernetes=$KUBERNETES"
|
|
fi
|
|
|
|
bazel run //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG"
|
|
|
|
- name: Remove Terraform plugin cache
|
|
if: always()
|
|
run: |
|
|
rm -rf constellation-terraform/.terraform
|
|
rm -rf constellation-iam-terraform/.terraform
|
|
|
|
- name: Upload Working Directory
|
|
if: always()
|
|
uses: ./.github/actions/artifact_upload
|
|
with:
|
|
name: constellation-post-test
|
|
path: |
|
|
${{ needs.create-cluster.outputs.kubeconfig }}
|
|
constellation-terraform
|
|
constellation-iam-terraform
|
|
constellation-conf.yaml
|
|
constellation-state.yaml
|
|
constellation-mastersecret.json
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
clean-up:
|
|
name: Clean up resources
|
|
runs-on: ubuntu-22.04
|
|
permissions:
|
|
id-token: write
|
|
checks: write
|
|
contents: read
|
|
packages: write
|
|
if: always()
|
|
needs: [generate-input-parameters, create-cluster, e2e-upgrade]
|
|
steps:
|
|
- name: Checkout
|
|
if: inputs.gitRef == 'head'
|
|
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
|
|
|
|
- name: Checkout ref
|
|
if: inputs.gitRef != 'head'
|
|
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
|
|
with:
|
|
fetch-depth: 0
|
|
ref: ${{ inputs.gitRef }}
|
|
|
|
- name: Download CLI
|
|
uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
|
|
with:
|
|
name: constellation
|
|
path: build
|
|
|
|
- name: Download Working Directory (Pre-test)
|
|
if: always() && needs.e2e-upgrade.result != 'success'
|
|
uses: ./.github/actions/artifact_download
|
|
with:
|
|
name: constellation-pre-test
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
- name: Download Working Directory (Post-test)
|
|
if: always() && needs.e2e-upgrade.result == 'success'
|
|
uses: ./.github/actions/artifact_download
|
|
with:
|
|
name: constellation-post-test
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
- name: Make Constellation executable and add to PATH
|
|
if: always()
|
|
run: |
|
|
chmod +x build/constellation
|
|
export PATH="$PATH:build"
|
|
echo "build" >> "$GITHUB_PATH"
|
|
|
|
- name: Always fetch logs
|
|
if: always()
|
|
env:
|
|
KUBECONFIG: ${{ needs.create-cluster.outputs.kubeconfig }}
|
|
run: |
|
|
kubectl logs -n kube-system -l "app.kubernetes.io/name=constellation-operator" --tail=-1 > node-operator.logs
|
|
kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs
|
|
kubectl get nodeversions.update.edgeless.systems constellation-version -o yaml > constellation-version.yaml
|
|
|
|
- name: Always upload logs
|
|
if: always()
|
|
uses: ./.github/actions/artifact_upload
|
|
with:
|
|
name: upgrade-logs
|
|
path: >
|
|
node-operator.logs
|
|
node-maintenance-operator.logs
|
|
constellation-version.yaml
|
|
encryptionSecret: ${{ secrets.ARTIFACT_ENCRYPT_PASSWD }}
|
|
|
|
- name: Always terminate cluster
|
|
if: always()
|
|
uses: ./.github/actions/constellation_destroy
|
|
with:
|
|
clusterCreation: "cli"
|
|
kubeconfig: ${{ needs.create-cluster.outputs.kubeconfig }}
|
|
cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
|
|
azureClusterDeleteCredentials: ${{ secrets.AZURE_E2E_CLUSTER_CREDENTIALS }}
|
|
gcpClusterDeleteServiceAccount: "infrastructure-e2e@constellation-e2e.iam.gserviceaccount.com"
|
|
|
|
- name: Always delete IAM configuration
|
|
if: always()
|
|
uses: ./.github/actions/constellation_iam_destroy
|
|
with:
|
|
cloudProvider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
|
|
azureCredentials: ${{ secrets.AZURE_E2E_IAM_CREDENTIALS }}
|
|
gcpServiceAccount: "iam-e2e@constellation-e2e.iam.gserviceaccount.com"
|
|
|
|
- name: Notify about failure
|
|
if: |
|
|
always() &&
|
|
( needs.create-cluster.result != 'success' || needs.e2e-upgrade.result != 'success' ) &&
|
|
github.ref == 'refs/heads/main' &&
|
|
inputs.scheduled
|
|
continue-on-error: true
|
|
uses: ./.github/actions/notify_e2e_failure
|
|
with:
|
|
projectWriteToken: ${{ secrets.PROJECT_WRITE_TOKEN }}
|
|
test: "upgrade"
|
|
provider: ${{ needs.generate-input-parameters.outputs.cloudProvider }}
|
|
attestationVariant: ${{ inputs.attestationVariant }}
|